diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 6d02c01ec5..60e83d9714 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -1,5 +1,5 @@ /** - * Copyright (C) 2025 Niklas Haas + * Copyright (C) 2025-2026 Niklas Haas * * This file is part of FFmpeg. * @@ -25,293 +25,27 @@ #include "libavutil/x86/cpu.h" #include "../ops_chain.h" +#include "../uops.h" +#include "../uops_macros.h" -#define DECL_ENTRY(TYPE, MASK, NAME, ...) \ - static const SwsOpEntry op_##NAME = { \ - .type = SWS_PIXEL_##TYPE, \ - .mask = MASK, \ - __VA_ARGS__ \ - } - -#define DECL_ASM(TYPE, MASK, NAME, ...) \ - void ff_##NAME(void); \ - DECL_ENTRY(TYPE, MASK, NAME, \ - .func = ff_##NAME, \ - __VA_ARGS__) - -#define DECL_PATTERN(TYPE, NAME, X, Y, Z, W, ...) \ - DECL_ASM(TYPE, SWS_COMP_MASK(X, Y, Z, W), p##X##Y##Z##W##_##NAME, \ - __VA_ARGS__ \ - ) - -#define REF_PATTERN(NAME, X, Y, Z, W) \ - &op_p##X##Y##Z##W##_##NAME - -#define DECL_COMMON_PATTERNS(TYPE, NAME, ...) \ - DECL_PATTERN(TYPE, NAME, 1, 0, 0, 0, __VA_ARGS__); \ - DECL_PATTERN(TYPE, NAME, 1, 0, 0, 1, __VA_ARGS__); \ - DECL_PATTERN(TYPE, NAME, 1, 1, 1, 0, __VA_ARGS__); \ - DECL_PATTERN(TYPE, NAME, 1, 1, 1, 1, __VA_ARGS__) \ - -#define REF_COMMON_PATTERNS(NAME) \ - REF_PATTERN(NAME, 1, 0, 0, 0), \ - REF_PATTERN(NAME, 1, 0, 0, 1), \ - REF_PATTERN(NAME, 1, 1, 1, 0), \ - REF_PATTERN(NAME, 1, 1, 1, 1) - -static int setup_rw(const SwsImplParams *params, SwsImplResult *out) +static int setup_rw_packed(const SwsImplParams *params, SwsImplResult *out) { - const SwsOp *op = params->op; + const SwsUOp *uop = params->uop; - /* 3-component reads/writes process one extra garbage word */ - if (op->rw.packed && op->rw.elems == 3) { - switch (op->op) { - case SWS_OP_READ: out->over_read = sizeof(uint32_t); break; - case SWS_OP_WRITE: out->over_write = sizeof(uint32_t); break; + /* 3-component packed reads/writes process one extra garbage word */ + if (uop->mask == SWS_COMP_ELEMS(3)) { + switch (uop->uop) { + case SWS_UOP_READ_PACKED: out->over_read = sizeof(uint32_t); break; + case SWS_UOP_WRITE_PACKED: out->over_write = sizeof(uint32_t); break; } } return 0; } -#define DECL_RW(EXT, TYPE, NAME, OP, ELEMS, PACKED, FRAC) \ - DECL_ASM(TYPE, SWS_COMP_ELEMS(ELEMS), NAME##ELEMS##EXT, \ - .op = SWS_OP_##OP, \ - .rw = { .elems = ELEMS, .packed = PACKED, .frac = FRAC }, \ - .setup = setup_rw, \ - ); - -#define DECL_PACKED_RW(EXT, DEPTH) \ - DECL_RW(EXT, U##DEPTH, read##DEPTH##_packed, READ, 2, true, 0) \ - DECL_RW(EXT, U##DEPTH, read##DEPTH##_packed, READ, 3, true, 0) \ - DECL_RW(EXT, U##DEPTH, read##DEPTH##_packed, READ, 4, true, 0) \ - DECL_RW(EXT, U##DEPTH, write##DEPTH##_packed, WRITE, 2, true, 0) \ - DECL_RW(EXT, U##DEPTH, write##DEPTH##_packed, WRITE, 3, true, 0) \ - DECL_RW(EXT, U##DEPTH, write##DEPTH##_packed, WRITE, 4, true, 0) \ - -#define DECL_PACK_UNPACK(EXT, TYPE, X, Y, Z, W) \ - DECL_ASM(TYPE, SWS_COMP(0), pack_##X##Y##Z##W##EXT, \ - .op = SWS_OP_PACK, \ - .pack.pattern = {X, Y, Z, W}, \ - ); \ - \ - DECL_ASM(TYPE, SWS_COMP_MASK(X, Y, Z, W), unpack_##X##Y##Z##W##EXT, \ - .op = SWS_OP_UNPACK, \ - .pack.pattern = {X, Y, Z, W}, \ - ); \ - -static int setup_swap_bytes(const SwsImplParams *params, SwsImplResult *out) -{ - const int mask = ff_sws_pixel_type_size(params->op->type) - 1; - for (int i = 0; i < 16; i++) - out->priv.u8[i] = (i & ~mask) | (mask - (i & mask)); - return 0; -} - -#define DECL_SWAP_BYTES(EXT, TYPE, X, Y, Z, W) \ - DECL_ENTRY(TYPE, SWS_COMP_MASK(X, Y, Z, W), \ - p##X##Y##Z##W##_swap_bytes_##TYPE##EXT, \ - .op = SWS_OP_SWAP_BYTES, \ - .func = ff_p##X##Y##Z##W##_shuffle##EXT, \ - .setup = setup_swap_bytes, \ - ); - -#define DECL_CLEAR_ALPHA(EXT, IDX) \ - DECL_ASM(U8, SWS_COMP_ALL, clear_alpha##IDX##EXT, \ - .op = SWS_OP_CLEAR, \ - .clear.mask = SWS_COMP(IDX), \ - .clear.value[IDX] = { -1, 1 }, \ - ); \ - -#define DECL_CLEAR_ZERO(EXT, IDX) \ - DECL_ASM(U8, SWS_COMP_ALL, clear_zero##IDX##EXT, \ - .op = SWS_OP_CLEAR, \ - .clear.mask = SWS_COMP(IDX), \ - .clear.value[IDX] = { 0, 1 }, \ - ); - -static int setup_clear(const SwsImplParams *params, SwsImplResult *out) -{ - const SwsOp *op = params->op; - for (int i = 0; i < 4; i++) - out->priv.u32[i] = (uint32_t) op->clear.value[i].num; - return 0; -} - -#define DECL_CLEAR(EXT, X, Y, Z, W) \ - DECL_ASM(U8, SWS_COMP_ALL, p##X##Y##Z##W##_clear##EXT, \ - .op = SWS_OP_CLEAR, \ - .setup = setup_clear, \ - .clear.mask = SWS_COMP_MASK(X, Y, Z, W), \ - ); - -#define DECL_SWIZZLE(EXT, X, Y, Z, W) \ - DECL_ASM(U8, SWS_COMP_ALL, swizzle_##X##Y##Z##W##EXT, \ - .op = SWS_OP_SWIZZLE, \ - .swizzle.in = {X, Y, Z, W}, \ - ); - -#define DECL_CONVERT(EXT, FROM, TO) \ - DECL_COMMON_PATTERNS(FROM, convert_##FROM##_##TO##EXT, \ - .op = SWS_OP_CONVERT, \ - .convert.to = SWS_PIXEL_##TO, \ - ); - -#define DECL_EXPAND(EXT, FROM, TO) \ - DECL_COMMON_PATTERNS(FROM, expand_##FROM##_##TO##EXT, \ - .op = SWS_OP_CONVERT, \ - .convert.to = SWS_PIXEL_##TO, \ - .convert.expand = true, \ - ); - -static int setup_shift(const SwsImplParams *params, SwsImplResult *out) -{ - out->priv.u16[0] = params->op->shift.amount; - return 0; -} - -#define DECL_SHIFT16(EXT) \ - DECL_COMMON_PATTERNS(U16, lshift16##EXT, \ - .op = SWS_OP_LSHIFT, \ - .setup = setup_shift, \ - .flexible = true, \ - ); \ - \ - DECL_COMMON_PATTERNS(U16, rshift16##EXT, \ - .op = SWS_OP_RSHIFT, \ - .setup = setup_shift, \ - .flexible = true, \ - ); - -#define DECL_MIN_MAX(EXT) \ - DECL_COMMON_PATTERNS(F32, min##EXT, \ - .op = SWS_OP_MIN, \ - .setup = ff_sws_setup_clamp, \ - ); \ - \ - DECL_COMMON_PATTERNS(F32, max##EXT, \ - .op = SWS_OP_MAX, \ - .setup = ff_sws_setup_clamp, \ - ); - -#define DECL_SCALE(EXT) \ - DECL_COMMON_PATTERNS(F32, scale##EXT, \ - .op = SWS_OP_SCALE, \ - .setup = ff_sws_setup_scale, \ - .flexible = true, \ - ); - -#define DECL_EXPAND_BITS(EXT, BITS) \ - DECL_ASM(U##BITS, SWS_COMP(0), expand_bits##BITS##EXT, \ - .op = SWS_OP_SCALE, \ - .scale = { .num = ((1 << (BITS)) - 1), .den = 1 }, \ - ); - -static int setup_dither(const SwsImplParams *params, SwsImplResult *out) -{ - const SwsOp *op = params->op; - const int size = 1 << op->dither.size_log2; - const int stride = size * sizeof(float); - const int8_t *off = op->dither.y_offset; - int max_offset = 0; - for (int i = 0; i < 4; i++) { - if (off[i] >= 0) - max_offset = FFMAX(max_offset, off[i] & (size - 1)); - } - - /* 1x1 matrix / single constant */ - if (!op->dither.size_log2) { - const AVRational k = op->dither.matrix[0]; - out->priv.f32[0] = (float) k.num / k.den; - goto store_offsets; - } - - /* Allocate extra rows to allow over-reading for row offsets. Note that - * max_offset is currently never larger than 5, so the extra space needed - * for this over-allocation is bounded by 5 * size * sizeof(float), - * typically 320 bytes for a 16x16 dither matrix. */ - const int num_rows = size + max_offset; - float *matrix = out->priv.ptr = av_mallocz(num_rows * stride); - if (!matrix) - return AVERROR(ENOMEM); - out->free = ff_op_priv_free; - - for (int i = 0; i < size * size; i++) - matrix[i] = (float) op->dither.matrix[i].num / op->dither.matrix[i].den; - - memcpy(&matrix[size * size], matrix, max_offset * stride); - -store_offsets: - /* Store relative pointer offset to each row inside extra space */ - static_assert(sizeof(out->priv.ptr) <= sizeof(int16_t[4]), - ">8 byte pointers not supported"); - assert(max_offset * stride <= INT16_MAX); - int16_t *off_out = &out->priv.i16[4]; - for (int i = 0; i < 4; i++) - off_out[i] = off[i] >= 0 ? (off[i] & (size - 1)) * stride : -1; - - return 0; -} - -#define DECL_DITHER(EXT, SIZE) \ - DECL_ASM(F32, SWS_COMP_ALL, dither##SIZE##EXT, \ - .op = SWS_OP_DITHER, \ - .setup = setup_dither, \ - .dither_size = SIZE, \ - ); - -static int setup_linear(const SwsImplParams *params, SwsImplResult *out) -{ - const SwsOp *op = params->op; - - float *matrix = out->priv.ptr = av_mallocz(sizeof(float[4][5])); - if (!matrix) - return AVERROR(ENOMEM); - out->free = ff_op_priv_free; - - for (int y = 0; y < 4; y++) { - for (int x = 0; x < 5; x++) - matrix[y * 5 + x] = (float) op->lin.m[y][x].num / op->lin.m[y][x].den; - } - - return 0; -} - -#define DECL_LINEAR(EXT, NAME, MASK) \ - DECL_ASM(F32, SWS_COMP_ALL, NAME##EXT, \ - .op = SWS_OP_LINEAR, \ - .setup = setup_linear, \ - .linear_mask = (MASK), \ - ); - -static bool check_filter_fma(const SwsImplParams *params) -{ - const SwsOp *op = params->op; - SwsContext *ctx = params->ctx; - if (!(ctx->flags & SWS_BITEXACT)) - return true; - - if (!ff_sws_pixel_type_is_int(op->type)) - return false; - - /* Check if maximum/minimum partial sum fits losslessly inside float */ - AVRational max_range = { 1 << 24, 1 }; - AVRational min_range = { -(1 << 24), 1 }; - const AVRational scale = Q(SWS_FILTER_SCALE); - - for (int i = 0; i < op->rw.elems; i++) { - const AVRational min = av_mul_q(op->comps.min[i], scale); - const AVRational max = av_mul_q(op->comps.max[i], scale); - if (av_cmp_q(min, min_range) < 0 || av_cmp_q(max_range, max) < 0) - return false; - } - - return true; -} - static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) { - const SwsFilterWeights *filter = params->op->rw.kernel; + const SwsFilterWeights *filter = params->uop->data.kernel; static_assert(sizeof(out->priv.ptr) <= sizeof(int32_t[2]), ">8 byte pointers not supported"); @@ -329,9 +63,9 @@ static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) return 0; } -static int hscale_sizeof_weight(const SwsOp *op) +static int hscale_sizeof_weight(const SwsUOp *uop) { - switch (op->type) { + switch (uop->type) { case SWS_PIXEL_U8: return sizeof(int16_t); case SWS_PIXEL_U16: return sizeof(int16_t); case SWS_PIXEL_F32: return sizeof(float); @@ -341,15 +75,15 @@ static int hscale_sizeof_weight(const SwsOp *op) static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) { - const SwsOp *op = params->op; - const SwsFilterWeights *filter = op->rw.kernel; + const SwsUOp *uop = params->uop; + const SwsFilterWeights *filter = uop->data.kernel; /** * `vpgatherdd` gathers 32 bits at a time; so if we're filtering a smaller * size, we need to gather 2/4 taps simultaneously and unroll the inner * loop over several packed samples. */ - const int pixel_size = ff_sws_pixel_type_size(op->type); + const int pixel_size = ff_sws_pixel_type_size(uop->type); const int taps_align = sizeof(int32_t) / pixel_size; const int filter_size = filter->filter_size; const int block_size = params->table->block_size; @@ -365,7 +99,7 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) float *f32; } weights; - const int sizeof_weight = hscale_sizeof_weight(op); + const int sizeof_weight = hscale_sizeof_weight(uop); weights.ptr = av_calloc(line_size, sizeof_weight * aligned_size); if (!weights.ptr) return AVERROR(ENOMEM); @@ -384,7 +118,7 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) for (int i = 0; i < elems; i++) { const int w = filter->weights[(x + i) * filter_size + j]; size_t idx = idx_base; - if (op->type == SWS_PIXEL_U8) { + if (uop->type == SWS_PIXEL_U8) { /* Interleave the pixels within each lane, i.e.: * [a0 a1 a2 a3 | b0 b1 b2 b3 ] pixels 0-1, taps 0-3 (lane 0) * [e0 e1 e2 e3 | f0 f1 f2 f3 ] pixels 4-5, taps 0-3 (lane 1) @@ -407,7 +141,7 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) idx += i * taps_align; } - switch (op->type) { + switch (uop->type) { case SWS_PIXEL_U8: weights.i16[idx] = w; break; case SWS_PIXEL_U16: weights.i16[idx] = w; break; case SWS_PIXEL_F32: weights.f32[idx] = w; break; @@ -423,11 +157,11 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) return 0; } -static bool check_filter_4x4_h(const SwsImplParams *params) +static bool check_filter_h_4x4(const SwsImplParams *params) { SwsContext *ctx = params->ctx; - const SwsOp *op = params->op; - if ((ctx->flags & SWS_BITEXACT) && op->type == SWS_PIXEL_F32) + const SwsUOp *uop = params->uop; + if ((ctx->flags & SWS_BITEXACT) && uop->type == SWS_PIXEL_F32) return false; /* different accumulation order due to 4x4 transpose */ const int cpu_flags = av_get_cpu_flags(); @@ -441,18 +175,18 @@ static bool check_filter_4x4_h(const SwsImplParams *params) * * Tested on a Lunar Lake (Intel Core Ultra 7 258V) system. */ - const SwsFilterWeights *filter = op->rw.kernel; - return op->type == SWS_PIXEL_U8 && filter->filter_size > 12 || - op->type == SWS_PIXEL_U16 && filter->filter_size > 4 || - op->type == SWS_PIXEL_F32 && filter->filter_size > 1; + const SwsFilterWeights *filter = uop->data.kernel; + return uop->type == SWS_PIXEL_U8 && filter->filter_size > 12 || + uop->type == SWS_PIXEL_U16 && filter->filter_size > 4 || + uop->type == SWS_PIXEL_F32 && filter->filter_size > 1; } -static int setup_filter_4x4_h(const SwsImplParams *params, SwsImplResult *out) +static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out) { - const SwsOp *op = params->op; - const SwsFilterWeights *filter = op->rw.kernel; - const int pixel_size = ff_sws_pixel_type_size(op->type); - const int sizeof_weights = hscale_sizeof_weight(op); + const SwsUOp *uop = params->uop; + const SwsFilterWeights *filter = uop->data.kernel; + const int pixel_size = ff_sws_pixel_type_size(uop->type); + const int sizeof_weights = hscale_sizeof_weight(uop); const int block_size = params->table->block_size; const int taps_align = 16 / sizeof_weights; /* taps per iteration (XMM) */ const int pixels_align = 4; /* pixels per iteration */ @@ -491,7 +225,7 @@ static int setup_filter_4x4_h(const SwsImplParams *params, SwsImplResult *out) const int w = filter->weights[x * filter_size + j]; const int idx = xb * aligned_size + jb * pixels_align + xi * taps_align + ji; - switch (op->type) { + switch (uop->type) { case SWS_PIXEL_U8: weights.i16[idx] = w; break; case SWS_PIXEL_U16: weights.i16[idx] = w; break; case SWS_PIXEL_F32: weights.f32[idx] = w; break; @@ -506,365 +240,246 @@ static int setup_filter_4x4_h(const SwsImplParams *params, SwsImplResult *out) return 0; } -#define DECL_FILTER(EXT, TYPE, DIR, NAME, ELEMS, ...) \ - DECL_ASM(TYPE, SWS_COMP_ELEMS(ELEMS), NAME##ELEMS##_##TYPE##EXT, \ - .op = SWS_OP_READ, \ - .rw.elems = ELEMS, \ - .rw.filter = SWS_OP_FILTER_##DIR, \ - __VA_ARGS__ \ - ); - -#define DECL_FILTERS(EXT, TYPE, DIR, NAME, ...) \ - DECL_FILTER(EXT, TYPE, DIR, NAME, 1, __VA_ARGS__) \ - DECL_FILTER(EXT, TYPE, DIR, NAME, 2, __VA_ARGS__) \ - DECL_FILTER(EXT, TYPE, DIR, NAME, 3, __VA_ARGS__) \ - DECL_FILTER(EXT, TYPE, DIR, NAME, 4, __VA_ARGS__) - -#define DECL_FILTERS_GENERIC(EXT, TYPE) \ - DECL_FILTERS(EXT, TYPE, V, filter_v, .setup = setup_filter_v) \ - DECL_FILTERS(EXT, TYPE, V, filter_fma_v, .setup = setup_filter_v, \ - .check = check_filter_fma) \ - DECL_FILTERS(EXT, TYPE, H, filter_h, .setup = setup_filter_h) \ - DECL_FILTERS(EXT, TYPE, H, filter_4x4_h, .setup = setup_filter_4x4_h, \ - .check = check_filter_4x4_h) - -#define REF_FILTERS(NAME, SUFFIX) \ - &op_##NAME##1##SUFFIX, \ - &op_##NAME##2##SUFFIX, \ - &op_##NAME##3##SUFFIX, \ - &op_##NAME##4##SUFFIX - -#define DECL_FUNCS_8(SIZE, EXT, FLAG) \ - DECL_RW(EXT, U8, read_planar, READ, 1, false, 0) \ - DECL_RW(EXT, U8, read_planar, READ, 2, false, 0) \ - DECL_RW(EXT, U8, read_planar, READ, 3, false, 0) \ - DECL_RW(EXT, U8, read_planar, READ, 4, false, 0) \ - DECL_RW(EXT, U8, write_planar, WRITE, 1, false, 0) \ - DECL_RW(EXT, U8, write_planar, WRITE, 2, false, 0) \ - DECL_RW(EXT, U8, write_planar, WRITE, 3, false, 0) \ - DECL_RW(EXT, U8, write_planar, WRITE, 4, false, 0) \ - DECL_RW(EXT, U8, read_nibbles, READ, 1, false, 1) \ - DECL_RW(EXT, U8, read_bits, READ, 1, false, 3) \ - DECL_RW(EXT, U8, write_bits, WRITE, 1, false, 3) \ - DECL_EXPAND_BITS(EXT, 8) \ - DECL_PACKED_RW(EXT, 8) \ - DECL_PACK_UNPACK(EXT, U8, 1, 2, 1, 0) \ - DECL_PACK_UNPACK(EXT, U8, 3, 3, 2, 0) \ - DECL_PACK_UNPACK(EXT, U8, 2, 3, 3, 0) \ - void ff_p1000_shuffle##EXT(void); \ - void ff_p1001_shuffle##EXT(void); \ - void ff_p1110_shuffle##EXT(void); \ - void ff_p1111_shuffle##EXT(void); \ - DECL_SWIZZLE(EXT, 3, 0, 1, 2) \ - DECL_SWIZZLE(EXT, 3, 0, 2, 1) \ - DECL_SWIZZLE(EXT, 2, 1, 0, 3) \ - DECL_SWIZZLE(EXT, 3, 2, 1, 0) \ - DECL_SWIZZLE(EXT, 3, 1, 0, 2) \ - DECL_SWIZZLE(EXT, 3, 2, 0, 1) \ - DECL_SWIZZLE(EXT, 1, 2, 0, 3) \ - DECL_SWIZZLE(EXT, 1, 0, 2, 3) \ - DECL_SWIZZLE(EXT, 2, 0, 1, 3) \ - DECL_SWIZZLE(EXT, 2, 3, 1, 0) \ - DECL_SWIZZLE(EXT, 2, 1, 3, 0) \ - DECL_SWIZZLE(EXT, 1, 2, 3, 0) \ - DECL_SWIZZLE(EXT, 1, 3, 2, 0) \ - DECL_SWIZZLE(EXT, 0, 2, 1, 3) \ - DECL_SWIZZLE(EXT, 0, 2, 3, 1) \ - DECL_SWIZZLE(EXT, 0, 3, 1, 2) \ - DECL_SWIZZLE(EXT, 3, 1, 2, 0) \ - DECL_SWIZZLE(EXT, 0, 3, 2, 1) \ - DECL_SWIZZLE(EXT, 0, 0, 0, 3) \ - DECL_SWIZZLE(EXT, 3, 0, 0, 0) \ - DECL_SWIZZLE(EXT, 0, 0, 0, 1) \ - DECL_SWIZZLE(EXT, 1, 0, 0, 0) \ - DECL_CLEAR_ALPHA(EXT, 0) \ - DECL_CLEAR_ALPHA(EXT, 1) \ - DECL_CLEAR_ALPHA(EXT, 3) \ - DECL_CLEAR_ZERO(EXT, 0) \ - DECL_CLEAR_ZERO(EXT, 1) \ - DECL_CLEAR_ZERO(EXT, 3) \ - DECL_CLEAR(EXT, 0, 0, 0, 1) \ - DECL_CLEAR(EXT, 1, 0, 0, 0) \ - DECL_CLEAR(EXT, 1, 1, 0, 0) \ - DECL_CLEAR(EXT, 0, 1, 1, 0) \ - DECL_CLEAR(EXT, 0, 0, 1, 1) \ - DECL_CLEAR(EXT, 1, 0, 1, 0) \ - DECL_CLEAR(EXT, 0, 1, 0, 1) \ - DECL_CLEAR(EXT, 0, 1, 1, 1) \ - DECL_CLEAR(EXT, 1, 0, 1, 1) \ - DECL_CLEAR(EXT, 1, 1, 0, 1) \ - \ -static const SwsOpTable ops8##EXT = { \ - .cpu_flags = AV_CPU_FLAG_##FLAG, \ - .block_size = SIZE, \ - .entries = { \ - &op_read_planar1##EXT, \ - &op_read_planar2##EXT, \ - &op_read_planar3##EXT, \ - &op_read_planar4##EXT, \ - &op_write_planar1##EXT, \ - &op_write_planar2##EXT, \ - &op_write_planar3##EXT, \ - &op_write_planar4##EXT, \ - &op_read8_packed2##EXT, \ - &op_read8_packed3##EXT, \ - &op_read8_packed4##EXT, \ - &op_write8_packed2##EXT, \ - &op_write8_packed3##EXT, \ - &op_write8_packed4##EXT, \ - &op_read_nibbles1##EXT, \ - &op_read_bits1##EXT, \ - &op_write_bits1##EXT, \ - &op_expand_bits8##EXT, \ - &op_pack_1210##EXT, \ - &op_pack_3320##EXT, \ - &op_pack_2330##EXT, \ - &op_unpack_1210##EXT, \ - &op_unpack_3320##EXT, \ - &op_unpack_2330##EXT, \ - &op_swizzle_3012##EXT, \ - &op_swizzle_3021##EXT, \ - &op_swizzle_2103##EXT, \ - &op_swizzle_3210##EXT, \ - &op_swizzle_3102##EXT, \ - &op_swizzle_3201##EXT, \ - &op_swizzle_1203##EXT, \ - &op_swizzle_1023##EXT, \ - &op_swizzle_2013##EXT, \ - &op_swizzle_2310##EXT, \ - &op_swizzle_2130##EXT, \ - &op_swizzle_1230##EXT, \ - &op_swizzle_1320##EXT, \ - &op_swizzle_0213##EXT, \ - &op_swizzle_0231##EXT, \ - &op_swizzle_0312##EXT, \ - &op_swizzle_3120##EXT, \ - &op_swizzle_0321##EXT, \ - &op_swizzle_0003##EXT, \ - &op_swizzle_0001##EXT, \ - &op_swizzle_3000##EXT, \ - &op_swizzle_1000##EXT, \ - &op_clear_alpha0##EXT, \ - &op_clear_alpha1##EXT, \ - &op_clear_alpha3##EXT, \ - &op_clear_zero0##EXT, \ - &op_clear_zero1##EXT, \ - &op_clear_zero3##EXT, \ - REF_PATTERN(clear##EXT, 0, 0, 0, 1), \ - REF_PATTERN(clear##EXT, 1, 0, 0, 0), \ - REF_PATTERN(clear##EXT, 1, 1, 0, 0), \ - REF_PATTERN(clear##EXT, 0, 1, 1, 0), \ - REF_PATTERN(clear##EXT, 0, 0, 1, 1), \ - REF_PATTERN(clear##EXT, 1, 0, 1, 0), \ - REF_PATTERN(clear##EXT, 0, 1, 0, 1), \ - REF_PATTERN(clear##EXT, 0, 1, 1, 1), \ - REF_PATTERN(clear##EXT, 1, 0, 1, 1), \ - REF_PATTERN(clear##EXT, 1, 1, 0, 1), \ - NULL \ - }, \ -}; - -#define DECL_FUNCS_16(SIZE, EXT, FLAG) \ - DECL_PACKED_RW(EXT, 16) \ - DECL_EXPAND_BITS(EXT, 16) \ - DECL_PACK_UNPACK(EXT, U16, 4, 4, 4, 0) \ - DECL_PACK_UNPACK(EXT, U16, 5, 5, 5, 0) \ - DECL_PACK_UNPACK(EXT, U16, 5, 6, 5, 0) \ - DECL_SWAP_BYTES(EXT, U16, 1, 0, 0, 0) \ - DECL_SWAP_BYTES(EXT, U16, 1, 0, 0, 1) \ - DECL_SWAP_BYTES(EXT, U16, 1, 1, 1, 0) \ - DECL_SWAP_BYTES(EXT, U16, 1, 1, 1, 1) \ - DECL_SHIFT16(EXT) \ - DECL_CONVERT(EXT, U8, U16) \ - DECL_CONVERT(EXT, U16, U8) \ - DECL_EXPAND(EXT, U8, U16) \ - \ -static const SwsOpTable ops16##EXT = { \ - .cpu_flags = AV_CPU_FLAG_##FLAG, \ - .block_size = SIZE, \ - .entries = { \ - &op_read16_packed2##EXT, \ - &op_read16_packed3##EXT, \ - &op_read16_packed4##EXT, \ - &op_write16_packed2##EXT, \ - &op_write16_packed3##EXT, \ - &op_write16_packed4##EXT, \ - &op_pack_4440##EXT, \ - &op_pack_5550##EXT, \ - &op_pack_5650##EXT, \ - &op_unpack_4440##EXT, \ - &op_unpack_5550##EXT, \ - &op_unpack_5650##EXT, \ - &op_expand_bits16##EXT, \ - REF_COMMON_PATTERNS(swap_bytes_U16##EXT), \ - REF_COMMON_PATTERNS(convert_U8_U16##EXT), \ - REF_COMMON_PATTERNS(convert_U16_U8##EXT), \ - REF_COMMON_PATTERNS(expand_U8_U16##EXT), \ - REF_COMMON_PATTERNS(lshift16##EXT), \ - REF_COMMON_PATTERNS(rshift16##EXT), \ - NULL \ - }, \ -}; - -#define DECL_FUNCS_32(SIZE, EXT, FLAG) \ - DECL_PACKED_RW(_m2##EXT, 32) \ - DECL_PACK_UNPACK(_m2##EXT, U32, 10, 10, 10, 2) \ - DECL_PACK_UNPACK(_m2##EXT, U32, 2, 10, 10, 10) \ - DECL_SWAP_BYTES(_m2##EXT, U32, 1, 0, 0, 0) \ - DECL_SWAP_BYTES(_m2##EXT, U32, 1, 0, 0, 1) \ - DECL_SWAP_BYTES(_m2##EXT, U32, 1, 1, 1, 0) \ - DECL_SWAP_BYTES(_m2##EXT, U32, 1, 1, 1, 1) \ - DECL_CONVERT(EXT, U8, U32) \ - DECL_CONVERT(EXT, U32, U8) \ - DECL_CONVERT(EXT, U16, U32) \ - DECL_CONVERT(EXT, U32, U16) \ - DECL_CONVERT(EXT, U8, F32) \ - DECL_CONVERT(EXT, F32, U8) \ - DECL_CONVERT(EXT, U16, F32) \ - DECL_CONVERT(EXT, F32, U16) \ - DECL_EXPAND(EXT, U8, U32) \ - DECL_MIN_MAX(EXT) \ - DECL_SCALE(EXT) \ - DECL_DITHER(EXT, 0) \ - DECL_DITHER(EXT, 1) \ - DECL_DITHER(EXT, 2) \ - DECL_DITHER(EXT, 3) \ - DECL_DITHER(EXT, 4) \ - DECL_DITHER(EXT, 5) \ - DECL_DITHER(EXT, 6) \ - DECL_DITHER(EXT, 7) \ - DECL_DITHER(EXT, 8) \ - DECL_LINEAR(EXT, luma, SWS_MASK_LUMA) \ - DECL_LINEAR(EXT, alpha, SWS_MASK_ALPHA) \ - DECL_LINEAR(EXT, lumalpha, SWS_MASK_LUMA | SWS_MASK_ALPHA) \ - DECL_LINEAR(EXT, yalpha, SWS_MASK(1, 1)) \ - DECL_LINEAR(EXT, dot3, 0x7) \ - DECL_LINEAR(EXT, dot3a, 0x7 | SWS_MASK_ALPHA) \ - DECL_LINEAR(EXT, row0, SWS_MASK_ROW(0) ^ SWS_MASK(0, 3)) \ - DECL_LINEAR(EXT, diag3, SWS_MASK_DIAG3) \ - DECL_LINEAR(EXT, diag4, SWS_MASK_DIAG4) \ - DECL_LINEAR(EXT, diagoff3, SWS_MASK_DIAG3 | SWS_MASK_OFF3) \ - DECL_LINEAR(EXT, affine3, SWS_MASK_MAT3 | SWS_MASK_OFF3) \ - DECL_LINEAR(EXT, affine3uv, \ - SWS_MASK_MAT3 | SWS_MASK_OFF(1) | SWS_MASK_OFF(2)) \ - DECL_LINEAR(EXT, affine3x, \ - SWS_MASK_MAT3 ^ SWS_MASK(0, 1) | SWS_MASK_OFF3) \ - DECL_LINEAR(EXT, affine3xa, \ - SWS_MASK_MAT3 ^ SWS_MASK(0, 1) | SWS_MASK_OFF3 | SWS_MASK_ALPHA) \ - DECL_LINEAR(EXT, affine3xy, \ - SWS_MASK_MAT3 ^ SWS_MASK(0, 0) ^ SWS_MASK(0, 1) | SWS_MASK_OFF3) \ - DECL_LINEAR(EXT, affine3a, \ - SWS_MASK_MAT3 | SWS_MASK_OFF3 | SWS_MASK_ALPHA) \ - DECL_FILTERS_GENERIC(EXT, U8) \ - DECL_FILTERS_GENERIC(EXT, U16) \ - DECL_FILTERS_GENERIC(EXT, F32) \ - \ -static const SwsOpTable ops32##EXT = { \ - .cpu_flags = AV_CPU_FLAG_##FLAG, \ - .block_size = SIZE, \ - .entries = { \ - &op_read32_packed2_m2##EXT, \ - &op_read32_packed3_m2##EXT, \ - &op_read32_packed4_m2##EXT, \ - &op_write32_packed2_m2##EXT, \ - &op_write32_packed3_m2##EXT, \ - &op_write32_packed4_m2##EXT, \ - &op_pack_1010102_m2##EXT, \ - &op_pack_2101010_m2##EXT, \ - &op_unpack_1010102_m2##EXT, \ - &op_unpack_2101010_m2##EXT, \ - REF_COMMON_PATTERNS(swap_bytes_U32_m2##EXT), \ - REF_COMMON_PATTERNS(convert_U8_U32##EXT), \ - REF_COMMON_PATTERNS(convert_U32_U8##EXT), \ - REF_COMMON_PATTERNS(convert_U16_U32##EXT), \ - REF_COMMON_PATTERNS(convert_U32_U16##EXT), \ - REF_COMMON_PATTERNS(convert_U8_F32##EXT), \ - REF_COMMON_PATTERNS(convert_F32_U8##EXT), \ - REF_COMMON_PATTERNS(convert_U16_F32##EXT), \ - REF_COMMON_PATTERNS(convert_F32_U16##EXT), \ - REF_COMMON_PATTERNS(expand_U8_U32##EXT), \ - REF_COMMON_PATTERNS(min##EXT), \ - REF_COMMON_PATTERNS(max##EXT), \ - REF_COMMON_PATTERNS(scale##EXT), \ - &op_dither0##EXT, \ - &op_dither1##EXT, \ - &op_dither2##EXT, \ - &op_dither3##EXT, \ - &op_dither4##EXT, \ - &op_dither5##EXT, \ - &op_dither6##EXT, \ - &op_dither7##EXT, \ - &op_dither8##EXT, \ - &op_luma##EXT, \ - &op_alpha##EXT, \ - &op_lumalpha##EXT, \ - &op_yalpha##EXT, \ - &op_dot3##EXT, \ - &op_dot3a##EXT, \ - &op_row0##EXT, \ - &op_diag3##EXT, \ - &op_diag4##EXT, \ - &op_diagoff3##EXT, \ - &op_affine3##EXT, \ - &op_affine3uv##EXT, \ - &op_affine3x##EXT, \ - &op_affine3xa##EXT, \ - &op_affine3xy##EXT, \ - &op_affine3a##EXT, \ - REF_FILTERS(filter_fma_v, _U8##EXT), \ - REF_FILTERS(filter_fma_v, _U16##EXT), \ - REF_FILTERS(filter_fma_v, _F32##EXT), \ - REF_FILTERS(filter_4x4_h, _U8##EXT), \ - REF_FILTERS(filter_4x4_h, _U16##EXT), \ - REF_FILTERS(filter_4x4_h, _F32##EXT), \ - REF_FILTERS(filter_v, _U8##EXT), \ - REF_FILTERS(filter_v, _U16##EXT), \ - REF_FILTERS(filter_v, _F32##EXT), \ - REF_FILTERS(filter_h, _U8##EXT), \ - REF_FILTERS(filter_h, _U16##EXT), \ - REF_FILTERS(filter_h, _F32##EXT), \ - NULL \ - }, \ -}; - -DECL_FUNCS_8(16, _m1_sse4, SSE4) -DECL_FUNCS_8(32, _m1_avx2, AVX2) -DECL_FUNCS_8(32, _m2_sse4, SSE4) -DECL_FUNCS_8(64, _m2_avx2, AVX2) - -DECL_FUNCS_16(16, _m1_avx2, AVX2) -DECL_FUNCS_16(32, _m2_avx2, AVX2) - -DECL_FUNCS_32(16, _avx2, AVX2) - -static const SwsOpTable *const tables[] = { - &ops8_m1_sse4, - &ops8_m1_avx2, - &ops8_m2_sse4, - &ops8_m2_avx2, - &ops16_m1_avx2, - &ops16_m2_avx2, - &ops32_avx2, -}; - -/** - * Returns true if the operation's implementation only depends on the block - * size, and not the underlying pixel type - */ -static bool op_is_type_invariant(const SwsOp *op) +static int setup_scale(const SwsImplParams *params, SwsImplResult *out) { - switch (op->op) { - case SWS_OP_READ: - case SWS_OP_WRITE: - return !(op->rw.elems > 1 && op->rw.packed) && !op->rw.frac && !op->rw.filter; - case SWS_OP_SWIZZLE: - case SWS_OP_CLEAR: - return true; + const SwsUOp *uop = params->uop; + switch (uop->type) { + case SWS_PIXEL_U8: out->priv.u16[0] = uop->data.scalar.u8; break; /* for pmullw */ + case SWS_PIXEL_U16: out->priv.u16[0] = uop->data.scalar.u16; break; + case SWS_PIXEL_U32: out->priv.u32[0] = uop->data.scalar.u32; break; + case SWS_PIXEL_F32: out->priv.f32[0] = uop->data.scalar.f32; break; + default: return AVERROR(EINVAL); } - return false; + return 0; } +static int setup_clear(const SwsImplParams *params, SwsImplResult *out) +{ + const SwsUOp *uop = params->uop; + for (int i = 0; i < 4; i++) + out->priv.u32[i] = uop->data.vec4[i].u32; + return 0; +} + +static int setup_dither(const SwsImplParams *params, SwsImplResult *out) +{ + out->priv.ptr = av_refstruct_ref(params->uop->data.ptr); + out->free = ff_op_priv_unref; + return 0; +} + +static int setup_linear(const SwsImplParams *params, SwsImplResult *out) +{ + const SwsUOp *uop = params->uop; + out->priv.ptr = av_memdup(uop->data.mat4, sizeof(uop->data.mat4)); + out->free = ff_op_priv_free; + return out->priv.ptr ? 0 : AVERROR(ENOMEM); +} + +static bool uop_is_type_invariant(const SwsUOpType uop) +{ + switch (uop) { + case SWS_UOP_READ_PLANAR: + case SWS_UOP_WRITE_PLANAR: + case SWS_UOP_CLEAR: + return true; + default: + return false; + } +} + +#define REF_ENTRY(EXT, NAME, ...) &op_##NAME##EXT, +#define DECL_ENTRY(EXT, CHECK, SETUP, NAME, ...) \ + void ff_##NAME##EXT(void); \ + static const SwsOpEntry op_##NAME##EXT = { \ + .func = (SwsFuncPtr) ff_##NAME##EXT, \ + .check = CHECK, \ + .setup = SETUP, \ + __VA_ARGS__, \ + }; + +/* Define all UOPs except conversion ops and type-invariant ops */ +#define DECL_OPS_COMMON(EXT, TYPE) \ +SWS_FOR_STRUCT(TYPE, READ_PACKED, DECL_ENTRY, EXT, NULL, setup_rw_packed) \ +SWS_FOR_STRUCT(TYPE, READ_NIBBLE, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, READ_BIT, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, WRITE_PACKED, DECL_ENTRY, EXT, NULL, setup_rw_packed) \ +SWS_FOR_STRUCT(TYPE, WRITE_NIBBLE, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, WRITE_BIT, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, SWAP_BYTES, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, EXPAND_BIT, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, PERMUTE, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, COPY, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, SCALE, DECL_ENTRY, EXT, NULL, setup_scale) \ +SWS_FOR_STRUCT(TYPE, ADD, DECL_ENTRY, EXT, NULL, ff_sws_setup_vec4) \ +SWS_FOR_STRUCT(TYPE, MIN, DECL_ENTRY, EXT, NULL, ff_sws_setup_vec4) \ +SWS_FOR_STRUCT(TYPE, MAX, DECL_ENTRY, EXT, NULL, ff_sws_setup_vec4) \ +SWS_FOR_STRUCT(TYPE, UNPACK, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, PACK, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, LSHIFT, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, RSHIFT, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(TYPE, LINEAR_FMA, DECL_ENTRY, EXT, NULL, setup_linear) \ +SWS_FOR_STRUCT(TYPE, DITHER, DECL_ENTRY, EXT, NULL, setup_dither) \ +/* end of macro */ + +#define REF_OPS_COMMON(EXT, TYPE) \ + SWS_FOR(TYPE, READ_PACKED, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, READ_NIBBLE, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, READ_BIT, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, WRITE_PACKED, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, WRITE_NIBBLE, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, WRITE_BIT, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, SWAP_BYTES, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, EXPAND_BIT, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, PERMUTE, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, COPY, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, SCALE, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, ADD, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, MIN, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, MAX, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, UNPACK, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, PACK, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, LSHIFT, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, RSHIFT, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, LINEAR_FMA, REF_ENTRY, EXT) \ + SWS_FOR(TYPE, DITHER, REF_ENTRY, EXT) \ + /* end of macro */ + +#define DECL_TABLE_U8(EXT, SIZE, FLAG) \ +DECL_OPS_COMMON(EXT, U8) \ +SWS_FOR_STRUCT(U8, READ_PLANAR, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U8, WRITE_PLANAR, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U8, CLEAR, DECL_ENTRY, EXT, NULL, setup_clear) \ + \ +static const SwsOpTable ops_u8##EXT = { \ + .cpu_flags = AV_CPU_FLAG_##FLAG, \ + .block_size = SIZE, \ + .uops = true, \ + .entries = { \ + REF_OPS_COMMON(EXT, U8) \ + SWS_FOR(U8, READ_PLANAR, REF_ENTRY, EXT) \ + SWS_FOR(U8, WRITE_PLANAR, REF_ENTRY, EXT) \ + SWS_FOR(U8, CLEAR, REF_ENTRY, EXT) \ + NULL \ + }, \ +}; + +#define DECL_TABLE_U16(EXT, SIZE, FLAG) \ +DECL_OPS_COMMON(EXT, U16) \ +SWS_FOR_STRUCT(U8, TO_U16, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U16, TO_U8, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U8, EXPAND_PAIR, DECL_ENTRY, EXT, NULL, NULL) \ + \ +static const SwsOpTable ops_u16##EXT = { \ + .cpu_flags = AV_CPU_FLAG_##FLAG, \ + .block_size = SIZE, \ + .uops = true, \ + .entries = { \ + REF_OPS_COMMON(EXT, U16) \ + SWS_FOR(U8, TO_U16, REF_ENTRY, EXT) \ + SWS_FOR(U16, TO_U8, REF_ENTRY, EXT) \ + SWS_FOR(U8, EXPAND_PAIR, REF_ENTRY, EXT) \ + NULL \ + }, \ +}; + +#define DECL_TABLE_U32(EXT, SIZE, FLAG) \ +DECL_OPS_COMMON(EXT, U32) \ +SWS_FOR_STRUCT(U8, TO_U32, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U32, TO_U8, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U16, TO_U32, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U32, TO_U16, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U8, EXPAND_QUAD, DECL_ENTRY, EXT, NULL, NULL) \ + \ +static const SwsOpTable ops_u32##EXT = { \ + .cpu_flags = AV_CPU_FLAG_##FLAG, \ + .block_size = SIZE, \ + .uops = true, \ + .entries = { \ + REF_OPS_COMMON(EXT, U32) \ + SWS_FOR(U8, TO_U32, REF_ENTRY, EXT) \ + SWS_FOR(U32, TO_U8, REF_ENTRY, EXT) \ + SWS_FOR(U16, TO_U32, REF_ENTRY, EXT) \ + SWS_FOR(U32, TO_U16, REF_ENTRY, EXT) \ + SWS_FOR(U8, EXPAND_QUAD, REF_ENTRY, EXT) \ + NULL \ + }, \ +}; + +#define DECL_TABLE_F32(EXT, SIZE, FLAG) \ +DECL_OPS_COMMON(EXT, F32) \ +SWS_FOR_STRUCT(U8, TO_F32, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(F32, TO_U8, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U16, TO_F32, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(F32, TO_U16, DECL_ENTRY, EXT, NULL, NULL) \ +SWS_FOR_STRUCT(U8, READ_PLANAR_FH, DECL_ENTRY, EXT, NULL, setup_filter_h) \ +SWS_FOR_STRUCT(U16, READ_PLANAR_FH, DECL_ENTRY, EXT, NULL, setup_filter_h) \ +SWS_FOR_STRUCT(F32, READ_PLANAR_FH, DECL_ENTRY, EXT, NULL, setup_filter_h) \ +SWS_FOR_STRUCT(U8, READ_PLANAR_FH, DECL_ENTRY, _4x4##EXT, \ + check_filter_h_4x4, setup_filter_h_4x4) \ +SWS_FOR_STRUCT(U16, READ_PLANAR_FH, DECL_ENTRY, _4x4##EXT, \ + check_filter_h_4x4, setup_filter_h_4x4) \ +SWS_FOR_STRUCT(F32, READ_PLANAR_FH, DECL_ENTRY, _4x4##EXT, \ + check_filter_h_4x4, setup_filter_h_4x4) \ +SWS_FOR_STRUCT(U8, READ_PLANAR_FV, DECL_ENTRY, EXT, NULL, setup_filter_v) \ +SWS_FOR_STRUCT(U16, READ_PLANAR_FV, DECL_ENTRY, EXT, NULL, setup_filter_v) \ +SWS_FOR_STRUCT(F32, READ_PLANAR_FV, DECL_ENTRY, EXT, NULL, setup_filter_v) \ +SWS_FOR_STRUCT(U8, READ_PLANAR_FV_FMA, DECL_ENTRY, EXT, NULL, setup_filter_v) \ +SWS_FOR_STRUCT(U16, READ_PLANAR_FV_FMA, DECL_ENTRY, EXT, NULL, setup_filter_v) \ +SWS_FOR_STRUCT(F32, READ_PLANAR_FV_FMA, DECL_ENTRY, EXT, NULL, setup_filter_v) \ + \ +static const SwsOpTable ops_f32##EXT = { \ + .cpu_flags = AV_CPU_FLAG_##FLAG, \ + .block_size = SIZE, \ + .uops = true, \ + .entries = { \ + REF_OPS_COMMON(EXT, F32) \ + SWS_FOR(U8, TO_F32, REF_ENTRY, EXT) \ + SWS_FOR(F32, TO_U8, REF_ENTRY, EXT) \ + SWS_FOR(U16, TO_F32, REF_ENTRY, EXT) \ + SWS_FOR(F32, TO_U16, REF_ENTRY, EXT) \ + SWS_FOR(U8, READ_PLANAR_FH, REF_ENTRY, _4x4##EXT) \ + SWS_FOR(U16, READ_PLANAR_FH, REF_ENTRY, _4x4##EXT) \ + SWS_FOR(F32, READ_PLANAR_FH, REF_ENTRY, _4x4##EXT) \ + SWS_FOR(U8, READ_PLANAR_FH, REF_ENTRY, EXT) \ + SWS_FOR(U16, READ_PLANAR_FH, REF_ENTRY, EXT) \ + SWS_FOR(F32, READ_PLANAR_FH, REF_ENTRY, EXT) \ + SWS_FOR(U8, READ_PLANAR_FV, REF_ENTRY, EXT) \ + SWS_FOR(U16, READ_PLANAR_FV, REF_ENTRY, EXT) \ + SWS_FOR(F32, READ_PLANAR_FV, REF_ENTRY, EXT) \ + SWS_FOR(U8, READ_PLANAR_FV_FMA, REF_ENTRY, EXT) \ + SWS_FOR(U16, READ_PLANAR_FV_FMA, REF_ENTRY, EXT) \ + SWS_FOR(F32, READ_PLANAR_FV_FMA, REF_ENTRY, EXT) \ + NULL \ + }, \ +}; + +DECL_TABLE_U8( _m1_sse4, 16, SSE4) +DECL_TABLE_U8( _m1_avx2, 32, AVX2) +DECL_TABLE_U8( _m2_sse4, 32, SSE4) +DECL_TABLE_U8( _m2_avx2, 64, AVX2) +DECL_TABLE_U16(_m1_avx2, 16, AVX2) +DECL_TABLE_U16(_m2_avx2, 32, AVX2) +DECL_TABLE_U32(_m2_avx2, 16, AVX2) +DECL_TABLE_F32(_m2_avx2, 16, AVX2) + +static const SwsOpTable *const tables[] = { + &ops_u8_m1_sse4, + &ops_u8_m1_avx2, /* order before _m2_sse4 */ + &ops_u8_m2_sse4, + &ops_u8_m2_avx2, + &ops_u16_m1_avx2, + &ops_u16_m2_avx2, + &ops_u32_m2_avx2, + &ops_f32_m2_avx2, +}; + +SWS_DECL_FUNC(ff_sws_process1_x86); +SWS_DECL_FUNC(ff_sws_process2_x86); +SWS_DECL_FUNC(ff_sws_process3_x86); +SWS_DECL_FUNC(ff_sws_process4_x86); + static int movsize(const int bytes, const int mmsize) { return bytes <= 4 ? 4 : /* movd */ @@ -936,32 +551,24 @@ do { return 0; } -/* Normalize clear values into 32-bit integer constants */ -static void normalize_clear(SwsOp *op) +/* Expand pixel value to 32-bits by repeating as necessary */ +static uint32_t expand32(const SwsPixelType type, const SwsPixel value) { - static_assert(sizeof(uint32_t) == sizeof(int), "int size mismatch"); - SwsImplResult res; - union { - uint32_t u32; - int i; - } c; - - ff_sws_setup_clear(&(const SwsImplParams) { .op = op }, &res); - - for (int i = 0; i < 4; i++) { - if (!SWS_COMP_TEST(op->clear.mask, i)) - continue; - switch (ff_sws_pixel_type_size(op->type)) { - case 1: c.u32 = 0x1010101U * res.priv.u8[i]; break; - case 2: c.u32 = (uint32_t) res.priv.u16[i] << 16 | res.priv.u16[i]; break; - case 4: c.u32 = res.priv.u32[i]; break; - } - - op->clear.value[i].num = c.i; - op->clear.value[i].den = 1; + switch (type) { + case SWS_PIXEL_U8: return value.u8 * 0x01010101u; + case SWS_PIXEL_U16: return value.u16 * 0x00010001u; + case SWS_PIXEL_U32: return value.u32; + case SWS_PIXEL_F32: return value.u32; /* reinterpret */ + default: return 0; } } +static void normalize_clear(SwsUOp *uop) +{ + for (int i = 0; i < 4; i++) + uop->data.vec4[i].u32 = expand32(uop->type, uop->data.vec4[i]); +} + static int compile(SwsContext *ctx, SwsOpList *ops, SwsCompiledOp *out) { const int cpu_flags = av_get_cpu_flags(); @@ -984,50 +591,54 @@ static int compile(SwsContext *ctx, SwsOpList *ops, SwsCompiledOp *out) if (!chain) return AVERROR(ENOMEM); - *out = (SwsCompiledOp) { - .priv = chain, - .slice_align = 1, - .free = ff_sws_op_chain_free_cb, - - /* Use at most two full YMM regs during the widest precision section */ - .block_size = 2 * FFMIN(mmsize, 32) / ff_sws_op_list_max_size(ops), - }; - - for (int i = 0; i < ops->num_ops; i++) { - int op_block_size = out->block_size; - SwsOp *op = &ops->ops[i]; - - if (op_is_type_invariant(op)) { - if (op->op == SWS_OP_CLEAR) - normalize_clear(op); - op_block_size *= ff_sws_pixel_type_size(op->type); - op->type = SWS_PIXEL_U8; - } - - ret = ff_sws_op_compile_tables(ctx, tables, FF_ARRAY_ELEMS(tables), - op, op_block_size, chain); - if (ret < 0) { - av_log(ctx, AV_LOG_TRACE, "Failed to compile op %d\n", i); - ff_sws_op_chain_free(chain); - return ret; - } + SwsUOpList *uops = ff_sws_uop_list_alloc(); + if (!uops) { + ret = AVERROR(ENOMEM); + goto fail; } -#define ASSIGN_PROCESS_FUNC(NAME) \ - do { \ - SWS_DECL_FUNC(NAME); \ - out->func = NAME; \ - } while (0) + SwsUOpFlags flags = 0; + if (X86_FMA4(cpu_flags)) + flags |= SWS_UOP_FLAG_FMA; + + ret = ff_sws_ops_translate(ctx, ops, flags, uops); + if (ret < 0) + goto fail; + + *out = (SwsCompiledOp) { + /* Use at most two full YMM regs during the widest precision section */ + .block_size = 2 * FFMIN(mmsize, 32) / ff_sws_op_list_max_size(ops), + .slice_align = 1, + .free = ff_sws_op_chain_free_cb, + .priv = chain, + }; + + for (int i = 0; i < uops->num_ops; i++) { + SwsUOp *uop = &uops->ops[i]; + int op_block_size = out->block_size; + + if (uop_is_type_invariant(uop->uop)) { + if (uop->uop == SWS_UOP_CLEAR) + normalize_clear(uop); + op_block_size *= ff_sws_pixel_type_size(uop->type); + uop->type = SWS_PIXEL_U8; + } + + ret = ff_sws_uop_lookup(ctx, tables, FF_ARRAY_ELEMS(tables), uop, + op_block_size, chain); + if (ret < 0) + goto fail; + } const SwsOp *read = ff_sws_op_list_input(ops); const SwsOp *write = ff_sws_op_list_output(ops); const int read_planes = read ? (read->rw.packed ? 1 : read->rw.elems) : 0; const int write_planes = write->rw.packed ? 1 : write->rw.elems; switch (FFMAX(read_planes, write_planes)) { - case 1: ASSIGN_PROCESS_FUNC(ff_sws_process1_x86); break; - case 2: ASSIGN_PROCESS_FUNC(ff_sws_process2_x86); break; - case 3: ASSIGN_PROCESS_FUNC(ff_sws_process3_x86); break; - case 4: ASSIGN_PROCESS_FUNC(ff_sws_process4_x86); break; + case 1: out->func = ff_sws_process1_x86; break; + case 2: out->func = ff_sws_process2_x86; break; + case 3: out->func = ff_sws_process3_x86; break; + case 4: out->func = ff_sws_process4_x86; break; } if (ret < 0) { @@ -1038,7 +649,13 @@ static int compile(SwsContext *ctx, SwsOpList *ops, SwsCompiledOp *out) out->cpu_flags = chain->cpu_flags; out->over_read = chain->over_read; out->over_write = chain->over_write; + ff_sws_uop_list_free(&uops); return 0; + +fail: + ff_sws_uop_list_free(&uops); + ff_sws_op_chain_free(chain); + return ret; } const SwsOpBackend backend_x86 = { diff --git a/libswscale/x86/ops_float.asm b/libswscale/x86/ops_float.asm index 605a9aa005..13f110b20b 100644 --- a/libswscale/x86/ops_float.asm +++ b/libswscale/x86/ops_float.asm @@ -1,5 +1,5 @@ ;****************************************************************************** -;* Copyright (c) 2025 Niklas Haas +;* Copyright (c) 2025-2026 Niklas Haas ;* ;* This file is part of FFmpeg. ;* @@ -19,465 +19,131 @@ ;****************************************************************************** %include "ops_include.asm" -%define SWS_FILTER_SCALE (1 << 14) + +%assign SWS_FILTER_SCALE (1 << 14) +%define FILTER_WIDTH (BLOCK_SIZE / 4) ; always outputs F32 +%define FILTER_SIZE (FILTER_WIDTH * BYTES) ; in source bytes SECTION_RODATA align 32 bias16: times 16 dw 0x8000 ; shift unsigned to signed range -bias32: times 8 dd 0x8000 * SWS_FILTER_SCALE -scale_inv: times 8 dd 0x38800000 ; 1.0f / SWS_FILTER_SCALE -; block_size = mmsize * 2 / sizeof(float) (two grouped registers) -%macro get_block_size 0 - %define block_size (mmsize >> 1) -%if mmsize == 64 - %define block_shift 5 -%elif mmsize == 32 - %define block_shift 4 -%elif mmsize == 16 - %define block_shift 3 -%elif mmsize == 8 - %define block_shift 2 -%else - %error "Unsupported mmsize" -%endif -%endmacro +align 16 +bias32: times 4 dd 0x8000 * SWS_FILTER_SCALE +scale_inv: times 4 dd 0x38800000 ; 1.0f / SWS_FILTER_SCALE SECTION .text ;--------------------------------------------------------- -; Pixel type conversions +; Generic vertical filtering (add+mul and fma) -%macro conv8to32f 0 -op convert_U8_F32 - LOAD_CONT tmp0q -IF X, vpsrldq xmx2, xmx, 8 -IF Y, vpsrldq xmy2, xmy, 8 -IF Z, vpsrldq xmz2, xmz, 8 -IF W, vpsrldq xmw2, xmw, 8 -IF X, pmovzxbd mx, xmx -IF Y, pmovzxbd my, xmy -IF Z, pmovzxbd mz, xmz -IF W, pmovzxbd mw, xmw -IF X, pmovzxbd mx2, xmx2 -IF Y, pmovzxbd my2, xmy2 -IF Z, pmovzxbd mz2, xmz2 -IF W, pmovzxbd mw2, xmw2 -IF X, vcvtdq2ps mx, mx -IF Y, vcvtdq2ps my, my -IF Z, vcvtdq2ps mz, mz -IF W, vcvtdq2ps mw, mw -IF X, vcvtdq2ps mx2, mx2 -IF Y, vcvtdq2ps my2, my2 -IF Z, vcvtdq2ps mz2, mz2 -IF W, vcvtdq2ps mw2, mw2 - CONTINUE tmp0q -%endmacro - -%macro conv16to32f 0 -op convert_U16_F32 - LOAD_CONT tmp0q -IF X, vextracti128 xmx2, mx, 1 -IF Y, vextracti128 xmy2, my, 1 -IF Z, vextracti128 xmz2, mz, 1 -IF W, vextracti128 xmw2, mw, 1 -IF X, pmovzxwd mx, xmx -IF Y, pmovzxwd my, xmy -IF Z, pmovzxwd mz, xmz -IF W, pmovzxwd mw, xmw -IF X, pmovzxwd mx2, xmx2 -IF Y, pmovzxwd my2, xmy2 -IF Z, pmovzxwd mz2, xmz2 -IF W, pmovzxwd mw2, xmw2 -IF X, vcvtdq2ps mx, mx -IF Y, vcvtdq2ps my, my -IF Z, vcvtdq2ps mz, mz -IF W, vcvtdq2ps mw, mw -IF X, vcvtdq2ps mx2, mx2 -IF Y, vcvtdq2ps my2, my2 -IF Z, vcvtdq2ps mz2, mz2 -IF W, vcvtdq2ps mw2, mw2 - CONTINUE tmp0q -%endmacro - -%macro conv32fto8 0 -op convert_F32_U8 - LOAD_CONT tmp0q -IF X, cvttps2dq mx, mx -IF Y, cvttps2dq my, my -IF Z, cvttps2dq mz, mz -IF W, cvttps2dq mw, mw -IF X, cvttps2dq mx2, mx2 -IF Y, cvttps2dq my2, my2 -IF Z, cvttps2dq mz2, mz2 -IF W, cvttps2dq mw2, mw2 -IF X, packusdw mx, mx2 -IF Y, packusdw my, my2 -IF Z, packusdw mz, mz2 -IF W, packusdw mw, mw2 -IF X, vextracti128 xmx2, mx, 1 -IF Y, vextracti128 xmy2, my, 1 -IF Z, vextracti128 xmz2, mz, 1 -IF W, vextracti128 xmw2, mw, 1 - vzeroupper -IF X, packuswb xmx, xmx2 -IF Y, packuswb xmy, xmy2 -IF Z, packuswb xmz, xmz2 -IF W, packuswb xmw, xmw2 -IF X, vpshufd xmx, xmx, q3120 -IF Y, vpshufd xmy, xmy, q3120 -IF Z, vpshufd xmz, xmz, q3120 -IF W, vpshufd xmw, xmw, q3120 - CONTINUE tmp0q -%endmacro - -%macro conv32fto16 0 -op convert_F32_U16 - LOAD_CONT tmp0q -IF X, cvttps2dq mx, mx -IF Y, cvttps2dq my, my -IF Z, cvttps2dq mz, mz -IF W, cvttps2dq mw, mw -IF X, cvttps2dq mx2, mx2 -IF Y, cvttps2dq my2, my2 -IF Z, cvttps2dq mz2, mz2 -IF W, cvttps2dq mw2, mw2 -IF X, packusdw mx, mx2 -IF Y, packusdw my, my2 -IF Z, packusdw mz, mz2 -IF W, packusdw mw, mw2 -IF X, vpermq mx, mx, q3120 -IF Y, vpermq my, my, q3120 -IF Z, vpermq mz, mz, q3120 -IF W, vpermq mw, mw, q3120 - CONTINUE tmp0q -%endmacro - -;--------------------------------------------------------- -; Arithmetic operations - -%macro min_max 0 -op min -IF X, vbroadcastss m8, [implq + SwsOpImpl.priv + 0] -IF Y, vbroadcastss m9, [implq + SwsOpImpl.priv + 4] -IF Z, vbroadcastss m10, [implq + SwsOpImpl.priv + 8] -IF W, vbroadcastss m11, [implq + SwsOpImpl.priv + 12] - LOAD_CONT tmp0q -IF X, minps mx, mx, m8 -IF Y, minps my, my, m9 -IF Z, minps mz, mz, m10 -IF W, minps mw, mw, m11 -IF X, minps mx2, m8 -IF Y, minps my2, m9 -IF Z, minps mz2, m10 -IF W, minps mw2, m11 - CONTINUE tmp0q - -op max -IF X, vbroadcastss m8, [implq + SwsOpImpl.priv + 0] -IF Y, vbroadcastss m9, [implq + SwsOpImpl.priv + 4] -IF Z, vbroadcastss m10, [implq + SwsOpImpl.priv + 8] -IF W, vbroadcastss m11, [implq + SwsOpImpl.priv + 12] - LOAD_CONT tmp0q -IF X, maxps mx, m8 -IF Y, maxps my, m9 -IF Z, maxps mz, m10 -IF W, maxps mw, m11 -IF X, maxps mx2, m8 -IF Y, maxps my2, m9 -IF Z, maxps mz2, m10 -IF W, maxps mw2, m11 - CONTINUE tmp0q -%endmacro - -%macro scale 0 -op scale - vbroadcastss m8, [implq + SwsOpImpl.priv] - LOAD_CONT tmp0q -IF X, mulps mx, m8 -IF Y, mulps my, m8 -IF Z, mulps mz, m8 -IF W, mulps mw, m8 -IF X, mulps mx2, m8 -IF Y, mulps my2, m8 -IF Z, mulps mz2, m8 -IF W, mulps mw2, m8 - CONTINUE tmp0q -%endmacro - -;--------------------------------------------------------- -; Dithering - -%macro dither_row 5 ; size_log2, comp_idx, matrix, out, out2 - mov tmp0w, [implq + SwsOpImpl.priv + (4 + %2) * 2] ; priv.u16[4 + i] - ; test is tmp0w < 0 - test tmp0w, tmp0w - js .skip%2 -%if %1 == 0 - addps %4, m8 - addps %5, m8 -%elif %1 == 1 - vbroadcastsd m8, [%3 + tmp0q] - addps %4, m8 - addps %5, m8 -%elif %1 == 2 - VBROADCASTI128 m8, [%3 + tmp0q] - addps %4, m8 - addps %5, m8 -%else - addps %4, [%3 + tmp0q] - addps %5, [%3 + tmp0q + mmsize * ((4 << %1) > mmsize)] -%endif -.skip%2: -%endmacro - -%macro dither 1 ; size_log2 -op dither%1 -%if %1 == 0 - ; dither offset is constant - vbroadcastss m8, [implq + SwsOpImpl.priv] -%else - ; dither matrix is stored indirectly at the private data address - mov tmp1q, [implq + SwsOpImpl.priv] - ; add y offset. note that for 2x2, we would only need to look at the - ; sign of `y`, but this special case is ignored for simplicity reasons - ; (and because the current upstream format code never generates matrices - ; that small) - mov tmp0d, yd - and tmp0d, (1 << %1) - 1 - shl tmp0d, %1 + 2 ; * sizeof(float) - add tmp1q, tmp0q - %if (4 << %1) > 2 * mmsize - ; need to add in x offset - mov tmp0d, bxd - shl tmp0d, 6 ; sizeof(float[16]) - and tmp0d, (4 << %1) - 1 - add tmp1q, tmp0q - %endif -%endif - dither_row %1, 0, tmp1q, mx, mx2 - dither_row %1, 1, tmp1q, my, my2 - dither_row %1, 2, tmp1q, mz, mz2 - dither_row %1, 3, tmp1q, mw, mw2 - CONTINUE -%endmacro - -%macro dither_fns 0 - dither 0 - dither 1 - dither 2 - dither 3 - dither 4 - dither 5 - dither 6 - dither 7 - dither 8 -%endmacro - -;--------------------------------------------------------- -; Linear transformations - -%xdefine MASK(I, J) (1 << (5 * (I) + (J))) -%xdefine MASK_OFF(I) MASK(I, 4) -%xdefine MASK_ROW(I) (0x1F << (5 * (I))) -%xdefine MASK_COL(J) (0x8421 << J) -%xdefine MASK_ALL (1 << 20) - 1 -%xdefine MASK_LUMA MASK(0, 0) | MASK_OFF(0) -%xdefine MASK_ALPHA MASK(3, 3) -%xdefine MASK_DIAG3 MASK(0, 0) | MASK(1, 1) | MASK(2, 2) -%xdefine MASK_OFF3 MASK_OFF(0) | MASK_OFF(1) | MASK_OFF(2) -%xdefine MASK_MAT3 MASK(0, 0) | MASK(0, 1) | MASK(0, 2) |\ - MASK(1, 0) | MASK(1, 1) | MASK(1, 2) |\ - MASK(2, 0) | MASK(2, 1) | MASK(2, 2) -%xdefine MASK_DIAG4 MASK_DIAG3 | MASK(3, 3) -%xdefine MASK_OFF4 MASK_OFF3 | MASK_OFF(3) -%xdefine MASK_MAT4 MASK_ALL & ~MASK_OFF4 - -%macro linear_row 7 ; res, x, y, z, w, row, mask -%define COL(J) ((%7) & MASK(%6, J)) ; true if mask contains component J -%define NOP(J) (J == %6 && !COL(J)) ; true if J is untouched input component - - ; load weights - IF COL(0), vbroadcastss m12, [tmp0q + %6 * 20 + 0] - IF COL(1), vbroadcastss m13, [tmp0q + %6 * 20 + 4] - IF COL(2), vbroadcastss m14, [tmp0q + %6 * 20 + 8] - IF COL(3), vbroadcastss m15, [tmp0q + %6 * 20 + 12] - - ; initialize result vector as appropriate - %if COL(4) ; offset - vbroadcastss %1, [tmp0q + %6 * 20 + 16] - %elif NOP(0) - ; directly reuse first component vector if possible - mova %1, %2 - %else - xorps %1, %1 - %endif - - IF COL(0), mulps m12, %2 - IF COL(1), mulps m13, %3 - IF COL(2), mulps m14, %4 - IF COL(3), mulps m15, %5 - IF COL(0), addps %1, m12 - IF NOP(0) && COL(4), addps %1, %2 ; first vector was not reused - IF COL(1), addps %1, m13 - IF NOP(1), addps %1, %3 - IF COL(2), addps %1, m14 - IF NOP(2), addps %1, %4 - IF COL(3), addps %1, m15 - IF NOP(3), addps %1, %5 -%endmacro - -%macro linear_inner 5 ; x, y, z, w, mask - %define ROW(I) ((%5) & MASK_ROW(I)) - IF1 ROW(0), linear_row m8, %1, %2, %3, %4, 0, %5 - IF1 ROW(1), linear_row m9, %1, %2, %3, %4, 1, %5 - IF1 ROW(2), linear_row m10, %1, %2, %3, %4, 2, %5 - IF1 ROW(3), linear_row m11, %1, %2, %3, %4, 3, %5 - IF ROW(0), mova %1, m8 - IF ROW(1), mova %2, m9 - IF ROW(2), mova %3, m10 - IF ROW(3), mova %4, m11 -%endmacro - -%macro linear_mask 2 ; name, mask -op %1 - mov tmp0q, [implq + SwsOpImpl.priv] ; address of matrix - linear_inner mx, my, mz, mw, %2 - linear_inner mx2, my2, mz2, mw2, %2 - CONTINUE -%endmacro - -; specialized functions for very simple cases -%macro linear_dot3 0 -op dot3 - mov tmp0q, [implq + SwsOpImpl.priv] - vbroadcastss m12, [tmp0q + 0] - vbroadcastss m13, [tmp0q + 4] - vbroadcastss m14, [tmp0q + 8] - LOAD_CONT tmp0q - mulps mx, m12 - mulps m8, my, m13 - mulps m9, mz, m14 - addps mx, m8 - addps mx, m9 - mulps mx2, m12 - mulps m10, my2, m13 - mulps m11, mz2, m14 - addps mx2, m10 - addps mx2, m11 - CONTINUE tmp0q -%endmacro - -%macro linear_fns 0 - linear_dot3 - linear_mask luma, MASK_LUMA - linear_mask alpha, MASK_ALPHA - linear_mask lumalpha, MASK_LUMA | MASK_ALPHA - linear_mask yalpha, MASK(1, 1) - linear_mask dot3a, 0x7 | MASK_ALPHA - linear_mask row0, MASK_ROW(0) ^ MASK(0, 3) - linear_mask diag3, MASK_DIAG3 - linear_mask diag4, MASK_DIAG4 - linear_mask diagoff3, MASK_DIAG3 | MASK_OFF3 - linear_mask affine3, MASK_MAT3 | MASK_OFF3 - linear_mask affine3uv, MASK_MAT3 | MASK_OFF(1) | MASK_OFF(2) - linear_mask affine3x, MASK_MAT3 ^ MASK(0, 1) | MASK_OFF3 - linear_mask affine3xa, MASK_MAT3 ^ MASK(0, 1) | MASK_OFF3 | MASK_ALPHA - linear_mask affine3xy, MASK_MAT3 ^ MASK(0, 0) ^ MASK(0, 1) | MASK_OFF3 - linear_mask affine3a, MASK_MAT3 | MASK_OFF3 | MASK_ALPHA -%endmacro - -;--------------------------------------------------------- -; Filtering / scaling - -%macro floadU8 2 ; dst, src +%macro FLOAD_SWS_PIXEL_U8 2 ; dst, src pmovzxbd %1, %2 vcvtdq2ps %1, %1 %endmacro -%macro floadU16 2 ; dst, src +%macro FLOAD_SWS_PIXEL_U16 2 ; dst, src pmovzxwd %1, %2 vcvtdq2ps %1, %1 %endmacro -%macro floadF32 2 ; dst, src +%macro FLOAD_SWS_PIXEL_U32 2 ; dst, src + %error TODO, no clean way to convert U32 to F32 on AVX2 +%endmacro + +%macro FLOAD_SWS_PIXEL_F32 2 ; dst, src movu %1, %2 %endmacro -%macro fmaccum 4 ; variant, dst, srcA, srcB -%ifidn %1, none - mulps %2, %3, %4 -%elifidn %1, fma_v - fmaddps %2, %3, %4, %2 -%else - mulps %3, %4 - addps %2, %3 -%endif +%macro ACCUM_MULADD 3 ; sum, src, weight + mulps %2, %3 + addps %1, %2 %endmacro -%macro filter_v_iter 4 ; elems, type, sizeof_type, variant - vbroadcastss m12, [weights] - fload%2 m8, [in0q] -IF %1 > 1, fload%2 m9, [in1q] -IF %1 > 2, fload%2 m10, [in2q] -IF %1 > 3, fload%2 m11, [in3q] - fmaccum %4, mx, m8, m12 -IF %1 > 1, fmaccum %4, my, m9, m12 -IF %1 > 2, fmaccum %4, mz, m10, m12 -IF %1 > 3, fmaccum %4, mw, m11, m12 - fload%2 m8, [in0q + (mmsize >> 2) * %3] -IF %1 > 1, fload%2 m9, [in1q + (mmsize >> 2) * %3] -IF %1 > 2, fload%2 m10, [in2q + (mmsize >> 2) * %3] -IF %1 > 3, fload%2 m11, [in3q + (mmsize >> 2) * %3] - fmaccum %4, mx2, m8, m12 -IF %1 > 1, fmaccum %4, my2, m9, m12 -IF %1 > 2, fmaccum %4, mz2, m10, m12 -IF %1 > 3, fmaccum %4, mw2, m11, m12 +%macro ACCUM_FMA 3 ; sum, src, weight + fmaddps %1, %2, %3, %1 %endmacro -%macro filter_v 4 ; elems, type, sizeof_type, variant -op filter_%4%1_%2 +%macro filter_v_iter 1 ; accum_fn + vbroadcastss m12, [weights] +IF X, FLOAD_PX m8, [in0q] +IF Y, FLOAD_PX m9, [in1q] +IF Z, FLOAD_PX m10, [in2q] +IF W, FLOAD_PX m11, [in3q] +IF X, %1 mx, m8, m12 +IF Y, %1 my, m9, m12 +IF Z, %1 mz, m10, m12 +IF W, %1 mw, m11, m12 +IF X, FLOAD_PX m8, [in0q + (FILTER_SIZE >> 1)] +IF Y, FLOAD_PX m9, [in1q + (FILTER_SIZE >> 1)] +IF Z, FLOAD_PX m10, [in2q + (FILTER_SIZE >> 1)] +IF W, FLOAD_PX m11, [in3q + (FILTER_SIZE >> 1)] +IF X, %1 mx2, m8, m12 +IF Y, %1 my2, m9, m12 +IF Z, %1 mz2, m10, m12 +IF W, %1 mw2, m11, m12 +%endmacro + +%macro read_planar_fv 1 ; accum_fn +%xdefine FLOAD_PX FLOAD_ %+ TYPE %xdefine weights tmp0q %xdefine fltsize tmp1q - mov weights, [implq + SwsOpImpl.priv] ; float *weights - mov fltsize, [implq + SwsOpImpl.priv + 8] ; size_t filter_size - ; weights += filter_size * y * sizeof(float) - mov tmp2q, fltsize - imul tmp2q, yq - lea weights, [weights + 4 * tmp2q] - filter_v_iter %1, %2, %3, none - dec fltsize - jz .done - push in0q -IF %1 > 1, push in1q -IF %1 > 2, push in2q -IF %1 > 3, push in3q + +assert_idn FILTER_TYPE, SWS_PIXEL_F32 +%assign SIZEOF_WEIGHT 4 + + mov weights, [implq + SwsOpImpl.priv] ; float *weights + mov fltsize, [implq + SwsOpImpl.priv + 8] ; size_t filter_size + ; weights += filter_size * y * sizeof(float) + mov tmp2q, fltsize + imul tmp2q, yq + lea weights, [weights + SIZEOF_WEIGHT * tmp2q] + filter_v_iter mulps + dec fltsize + jz .done +IF X, push in0q +IF Y, push in1q +IF Z, push in2q +IF W, push in3q .loop: - add in0q, [execq + SwsOpExec.in_stride0] -IF %1 > 1, add in1q, [execq + SwsOpExec.in_stride1] -IF %1 > 2, add in2q, [execq + SwsOpExec.in_stride2] -IF %1 > 3, add in3q, [execq + SwsOpExec.in_stride3] - add weights, 4 - filter_v_iter %1, %2, %3, %4 - dec fltsize - jnz .loop -IF %1 > 3, pop in3q -IF %1 > 2, pop in2q -IF %1 > 1, pop in1q - pop in0q +IF X, add in0q, [execq + SwsOpExec.in_stride0] +IF Y, add in1q, [execq + SwsOpExec.in_stride1] +IF Z, add in2q, [execq + SwsOpExec.in_stride2] +IF W, add in3q, [execq + SwsOpExec.in_stride3] + add weights, SIZEOF_WEIGHT + filter_v_iter %1 + dec fltsize + jnz .loop +IF W, pop in3q +IF Z, pop in2q +IF Y, pop in1q +IF X, pop in0q .done: - LOAD_CONT tmp0q -IF %1 > 3, add in3q, (mmsize >> 1) * %3 -IF %1 > 2, add in2q, (mmsize >> 1) * %3 -IF %1 > 1, add in1q, (mmsize >> 1) * %3 - add in0q, (mmsize >> 1) * %3 - CONTINUE tmp0q + LOAD_CONT tmp0q +IF W, add in3q, FILTER_SIZE +IF Z, add in2q, FILTER_SIZE +IF Y, add in1q, FILTER_SIZE +IF X, add in0q, FILTER_SIZE + CONTINUE tmp0q %undef weights %undef fltsize %endmacro -%macro filter_h_iter_U8 4 ; acc, acc2, src, first +%macro READ_PLANAR_FV 1 +%xdefine FILTER_TYPE %1 + read_planar_fv ACCUM_MULADD +%endmacro + +%macro READ_PLANAR_FV_FMA 1 +%xdefine FILTER_TYPE %1 + read_planar_fv ACCUM_FMA +%endmacro + +;--------------------------------------------------------- +; Generic horizontal filtering (vpgatherdd) + +%macro FILTER_H_SWS_PIXEL_U8 4 ; acc, acc2, src, first pcmpeqb m12, m12 pcmpeqb m13, m13 vpgatherdd m8, [%3 + m14], m12 ; { ABCD | EFGH } 4 pixel per word @@ -491,246 +157,254 @@ IF %1 > 1, add in1q, (mmsize >> 1) * %3 pmaddwd m10, [weights + mmsize] pmaddwd m9, [weights + mmsize * 2] pmaddwd m11, [weights + mmsize * 3] -%if %4 + %if %4 phaddd %1, m8, m10 ; { ABCD | EFGH } phaddd %2, m9, m11 ; { IJKL | MNOP } -%else + %else phaddd m8, m10 phaddd m9, m11 paddd %1, m8 paddd %2, m9 -%endif + %endif %endmacro -%macro filter_h_iter_U16 4 ; acc, acc2, src, first +%macro FILTER_H_SWS_PIXEL_U16 4 ; acc, acc2, src, first pcmpeqb m12, m12 pcmpeqb m13, m13 vpgatherdd m8, [%3 + m14], m12 vpgatherdd m9, [%3 + m15], m13 psubw m8, m10 psubw m9, m10 -%if %4 + %if %4 pmaddwd %1, m8, [weights] pmaddwd %2, m9, [weights + mmsize] -%else + %else pmaddwd m8, [weights] pmaddwd m9, [weights + mmsize] paddd %1, m8 paddd %2, m9 -%endif + %endif %endmacro -%macro filter_h_iter_F32 4 ; acc, acc2, src, first +%macro FILTER_H_SWS_PIXEL_F32 4 ; acc, acc2, src, first pcmpeqb m12, m12 pcmpeqb m13, m13 vpgatherdd m8, [%3 + m14], m12 vpgatherdd m9, [%3 + m15], m13 -%if %4 + %if %4 mulps %1, m8, [weights] mulps %2, m9, [weights + mmsize] -%else + %else mulps m8, [weights] mulps m9, [weights + mmsize] addps %1, m8 addps %2, m9 -%endif + %endif %endmacro -%macro filter_h 4 ; elems, type, sizeof_type, sizeof_weight -op filter_h%1_%2 +%macro READ_PLANAR_FH 1 +%xdefine FILTER_TYPE %1 %xdefine weights tmp0q %xdefine fltsize tmp1q - mov tmp0q, [execq + SwsOpExec.in_offset_x] - mov fltsize, [implq + SwsOpImpl.priv + 8] ; size_t filter_size - get_block_size - mov tmp2d, bxd - shl tmp2q, block_shift ; x := bx * block_size - movu m14, [tmp0q + 4 * tmp2q] ; &exec->in_offset_x[x] - movu m15, [tmp0q + 4 * tmp2q + mmsize] - mov weights, [implq + SwsOpImpl.priv] -%ifidn %2, U16 - mova m10, [bias16] - mova m11, [bias32] +%xdefine FILTER_H_PX FILTER_H_ %+ TYPE + +assert_idn FILTER_TYPE, SWS_PIXEL_F32 +%ifidn TYPE, SWS_PIXEL_F32 + %assign SIZEOF_WEIGHT 4 ; F32 +%else + %assign SIZEOF_WEIGHT 2 ; I16 %endif - imul tmp2q, fltsize - lea weights, [weights + tmp2q * %4] ; weights += x * filter_size - filter_h_iter_%2 mx, mx2, in0q, 1 -IF1 %1 > 1, filter_h_iter_%2 my, my2, in1q, 1 -IF1 %1 > 2, filter_h_iter_%2 mz, mz2, in2q, 1 -IF1 %1 > 3, filter_h_iter_%2 mw, mw2, in3q, 1 - sub fltsize, 4 / %3 - jz .done - push in0q -IF %1 > 1, push in1q -IF %1 > 2, push in2q -IF %1 > 3, push in3q + + mov tmp0q, [execq + SwsOpExec.in_offset_x] + mov fltsize, [implq + SwsOpImpl.priv + 8] ; size_t filter_size + mov tmp2d, bxd + shl_log2 tmp2q, FILTER_WIDTH ; x := bx * FILTER_WIDTH + movu m14, [tmp0q + 4 * tmp2q] ; &exec->in_offset_x[x] + movu m15, [tmp0q + 4 * tmp2q + mmsize] + mov weights, [implq + SwsOpImpl.priv] + %ifidn TYPE, SWS_PIXEL_U16 + VBROADCASTI128 m10, [bias16] + VBROADCASTI128 m11, [bias32] + %endif + imul tmp2q, fltsize + lea weights, [weights + tmp2q * SIZEOF_WEIGHT] ; weights += x * filter_size +IF X, FILTER_H_PX mx, mx2, in0q, 1 +IF Y, FILTER_H_PX my, my2, in1q, 1 +IF Z, FILTER_H_PX mz, mz2, in2q, 1 +IF W, FILTER_H_PX mw, mw2, in3q, 1 + sub fltsize, 4 / BYTES + jz .done +IF X, push in0q +IF Y, push in1q +IF Z, push in2q +IF W, push in3q .loop: - add in0q, 4 -IF %1 > 1, add in1q, 4 -IF %1 > 2, add in2q, 4 -IF %1 > 3, add in3q, 4 - add weights, mmsize * 2 * (%4 / %3) - filter_h_iter_%2 mx, mx2, in0q, 0 -IF1 %1 > 1, filter_h_iter_%2 my, my2, in1q, 0 -IF1 %1 > 2, filter_h_iter_%2 mz, mz2, in2q, 0 -IF1 %1 > 3, filter_h_iter_%2 mw, mw2, in3q, 0 - sub fltsize, 4 / %3 - jnz .loop -IF %1 > 3, pop in3q -IF %1 > 2, pop in2q -IF %1 > 1, pop in1q - pop in0q +IF X, add in0q, 4 +IF Y, add in1q, 4 +IF Z, add in2q, 4 +IF W, add in3q, 4 + add weights, BLOCK_WIDTH * SIZEOF_WEIGHT +IF X, FILTER_H_PX mx, mx2, in0q, 0 +IF Y, FILTER_H_PX my, my2, in1q, 0 +IF Z, FILTER_H_PX mz, mz2, in2q, 0 +IF W, FILTER_H_PX mw, mw2, in3q, 0 + sub fltsize, 4 / BYTES + jnz .loop +IF W, pop in3q +IF Z, pop in2q +IF Y, pop in1q +IF X, pop in0q .done: -%ifidn %2, U16 - paddd mx, m11 -IF %1 > 1, paddd my, m11 -IF %1 > 2, paddd mz, m11 -IF %1 > 3, paddd mw, m11 - paddd mx2, m11 -IF %1 > 1, paddd my2, m11 -IF %1 > 2, paddd mz2, m11 -IF %1 > 3, paddd mw2, m11 -%endif -%ifnidn %2, F32 - vcvtdq2ps mx, mx -IF %1 > 1, vcvtdq2ps my, my -IF %1 > 2, vcvtdq2ps mz, mz -IF %1 > 3, vcvtdq2ps mw, mw - vcvtdq2ps mx2, mx2 -IF %1 > 1, vcvtdq2ps my2, my2 -IF %1 > 2, vcvtdq2ps mz2, mz2 -IF %1 > 3, vcvtdq2ps mw2, mw2 -%endif - mova m12, [scale_inv] - LOAD_CONT tmp0q - mulps mx, m12 -IF %1 > 1, mulps my, m12 -IF %1 > 2, mulps mz, m12 -IF %1 > 3, mulps mw, m12 - mulps mx2, m12 -IF %1 > 1, mulps my2, m12 -IF %1 > 2, mulps mz2, m12 -IF %1 > 3, mulps mw2, m12 - CONTINUE tmp0q + %ifidn TYPE, SWS_PIXEL_U16 +IF X, paddd mx, m11 +IF Y, paddd my, m11 +IF Z, paddd mz, m11 +IF W, paddd mw, m11 +IF X, paddd mx2, m11 +IF Y, paddd my2, m11 +IF Z, paddd mz2, m11 +IF W, paddd mw2, m11 + %endif + %ifnidn TYPE, SWS_PIXEL_F32 +IF X, vcvtdq2ps mx, mx +IF Y, vcvtdq2ps my, my +IF Z, vcvtdq2ps mz, mz +IF W, vcvtdq2ps mw, mw +IF X, vcvtdq2ps mx2, mx2 +IF Y, vcvtdq2ps my2, my2 +IF Z, vcvtdq2ps mz2, mz2 +IF W, vcvtdq2ps mw2, mw2 + %endif + VBROADCASTI128 m12, [scale_inv] + LOAD_CONT tmp0q +IF X, mulps mx, m12 +IF Y, mulps my, m12 +IF Z, mulps mz, m12 +IF W, mulps mw, m12 +IF X, mulps mx2, m12 +IF Y, mulps my2, m12 +IF Z, mulps mz2, m12 +IF W, mulps mw2, m12 + CONTINUE tmp0q %undef weights %undef fltsize %endmacro -%macro iloadU8 2 ; dst, src +;--------------------------------------------------------- +; 4x4 transposed horizontal filtering + +%macro ILOAD_SWS_PIXEL_U8 2 ; dst, src pmovzxbw %1, %2 %endmacro -%macro iloadU16 2 ; dst, src +%macro ILOAD_SWS_PIXEL_U16 2 ; dst, src movu %1, %2 psubw %1, [bias16] ; shift into signed I16 range %endmacro -%macro iloadF32 2 ; dst, src +%macro ILOAD_SWS_PIXEL_F32 2 ; dst, src movu %1, %2 %endmacro ; filter 4 adjacent pixels at the same time -%macro filter_h4 4 ; dst, src, type, sizeof_type -%ifidn %3, F32 +%macro filter_h4_4x4 2 ; dst, src +%ifidn TYPE, SWS_PIXEL_F32 %xdefine MUL mulps %xdefine ADD addps %else %xdefine MUL pmaddwd %xdefine ADD paddd %endif - iload%3 xm8, [%2 + offset0q] ; {a0, a1, a2, a3} - iload%3 xm9, [%2 + offset1q] ; {b0, b1, b2, b3} - iload%3 xm10, [%2 + offset2q] ; {c0, c1, c2, c3} - iload%3 xm11, [%2 + offset3q] ; {d0, d1, d2, d3} - MUL xm8, [weights] - MUL xm9, [weights + 16] - MUL xm10, [weights + 32] - MUL xm11, [weights + 48] - mov bxq, fltsize - sub bxq, 64 - jz %%done - push weights - push %2 + + ILOAD_PX xm8, [%2 + offset0q] ; {a0, a1, a2, a3} + ILOAD_PX xm9, [%2 + offset1q] ; {b0, b1, b2, b3} + ILOAD_PX xm10, [%2 + offset2q] ; {c0, c1, c2, c3} + ILOAD_PX xm11, [%2 + offset3q] ; {d0, d1, d2, d3} + MUL xm8, [weights] + MUL xm9, [weights + 16] + MUL xm10, [weights + 32] + MUL xm11, [weights + 48] + mov bxq, fltsize + sub bxq, 64 + jz %%done + push weights + push %2 %%loop: - add weights, 64 -%ifidn %3, F32 - add %2, 4 * %4 -%else - add %2, 8 * %4 -%endif - iload%3 xm14, [%2 + offset0q] - iload%3 xm15, [%2 + offset1q] - MUL xm14, [weights] - MUL xm15, [weights + 16] - ADD xm8, xm14 - ADD xm9, xm15 - iload%3 xm14, [%2 + offset2q] - iload%3 xm15, [%2 + offset3q] - MUL xm14, [weights + 32] - MUL xm15, [weights + 48] - ADD xm10, xm14 - ADD xm11, xm15 - sub bxq, 64 - jnz %%loop - pop %2 - pop weights + add weights, 64 + add %2, (16 / SIZEOF_WEIGHT) * BYTES ; pixels per xmm reg + ILOAD_PX xm14, [%2 + offset0q] + ILOAD_PX xm15, [%2 + offset1q] + MUL xm14, [weights] + MUL xm15, [weights + 16] + ADD xm8, xm14 + ADD xm9, xm15 + ILOAD_PX xm14, [%2 + offset2q] + ILOAD_PX xm15, [%2 + offset3q] + MUL xm14, [weights + 32] + MUL xm15, [weights + 48] + ADD xm10, xm14 + ADD xm11, xm15 + sub bxq, 64 + jnz %%loop + pop %2 + pop weights %%done: - ; 4x4 transpose (on XMM size) - punpckhdq xm15, xm8, xm9 ; {a2, b2, a3, b3} - punpckldq xm8, xm9 ; {a0, b0, a1, b1} - punpckhdq xm9, xm10, xm11 ; {c2, d2, c3, d3} - punpckldq xm10, xm11 ; {c0, d0, c1, d1} - punpckhqdq xm11, xm8, xm10 ; {a1, b1, c1, d1} - punpcklqdq xm8, xm10 ; {a0, b0, c0, d0} - punpckhqdq xm10, xm15, xm9 ; {a3, b3, c3, d3} - punpcklqdq xm15, xm9 ; {a2, b2, c2, d2} - ADD xm8, xm11 ; sum all even terms - ADD xm15, xm10 ; sum all odd terms - ADD %1, xm8, xm15 + ; 4x4 transpose (on XMM size) + punpckhdq xm15, xm8, xm9 ; {a2, b2, a3, b3} + punpckldq xm8, xm9 ; {a0, b0, a1, b1} + punpckhdq xm9, xm10, xm11 ; {c2, d2, c3, d3} + punpckldq xm10, xm11 ; {c0, d0, c1, d1} + punpckhqdq xm11, xm8, xm10 ; {a1, b1, c1, d1} + punpcklqdq xm8, xm10 ; {a0, b0, c0, d0} + punpckhqdq xm10, xm15, xm9 ; {a3, b3, c3, d3} + punpcklqdq xm15, xm9 ; {a2, b2, c2, d2} + ADD xm8, xm11 ; sum all even terms + ADD xm15, xm10 ; sum all odd terms + ADD %1, xm8, xm15 %undef MUL %undef ADD %endmacro -; filter low and high lines separately and combine results for each plane -%macro filter_h8 4-5 ; elems, type, sizeof_type, offsets, dst_suffix - movsxd offset0q, dword [%4 + 0] - movsxd offset1q, dword [%4 + 4] - movsxd offset2q, dword [%4 + 8] - movsxd offset3q, dword [%4 + 12] - filter_h4 xmx%5, in0q, %2, %3 -IF %1 > 1, filter_h4 xmy%5, in1q, %2, %3 -IF %1 > 2, filter_h4 xmz%5, in2q, %2, %3 -IF %1 > 3, filter_h4 xmw%5, in3q, %2, %3 - add weights, fltsize - movsxd offset0q, dword [%4 + 16] - movsxd offset1q, dword [%4 + 20] - movsxd offset2q, dword [%4 + 24] - movsxd offset3q, dword [%4 + 28] - filter_h4 xm12, in0q, %2, %3 -IF %1 > 1, filter_h4 xm13, in1q, %2, %3 - vinsertf128 mx%5, mx%5, xmm12, 1 -IF %1 > 1, vinsertf128 my%5, my%5, xmm13, 1 -IF %1 > 2, filter_h4 xm12, in2q, %2, %3 -IF %1 > 3, filter_h4 xm13, in3q, %2, %3 -IF %1 > 2, vinsertf128 mz%5, mz%5, xmm12, 1 -IF %1 > 3, vinsertf128 mw%5, mw%5, xmm13, 1 -%ifidn %2, U16 - mova m15, [bias32] - paddd mx%5, m15 -IF %1 > 1, paddd my%5, m15 -IF %1 > 2, paddd mz%5, m15 -IF %1 > 3, paddd mw%5, m15 -%endif -%ifnidn %2, F32 - vcvtdq2ps mx%5, mx%5 -IF %1 > 1, vcvtdq2ps my%5, my%5 -IF %1 > 2, vcvtdq2ps mz%5, mz%5 -IF %1 > 3, vcvtdq2ps mw%5, mw%5 -%endif +; filter low and high lanes separately and combine results for each plane +%macro filter_h8_4x4 1-2 ; offsets, dst_suffix + movsxd offset0q, dword [%1 + 0] + movsxd offset1q, dword [%1 + 4] + movsxd offset2q, dword [%1 + 8] + movsxd offset3q, dword [%1 + 12] +IF X, filter_h4_4x4 xmx%2, in0q +IF Y, filter_h4_4x4 xmy%2, in1q +IF Z, filter_h4_4x4 xmz%2, in2q +IF W, filter_h4_4x4 xmw%2, in3q + add weights, fltsize + movsxd offset0q, dword [%1 + 16] + movsxd offset1q, dword [%1 + 20] + movsxd offset2q, dword [%1 + 24] + movsxd offset3q, dword [%1 + 28] +IF X, filter_h4_4x4 xm12, in0q +IF Y, filter_h4_4x4 xm13, in1q +IF X, vinsertf128 mx%2, mx%2, xmm12, 1 +IF Y, vinsertf128 my%2, my%2, xmm13, 1 +IF Z, filter_h4_4x4 xm12, in2q +IF W, filter_h4_4x4 xm13, in3q +IF Z, vinsertf128 mz%2, mz%2, xmm12, 1 +IF W, vinsertf128 mw%2, mw%2, xmm13, 1 + %ifidn TYPE, SWS_PIXEL_U16 + VBROADCASTI128 m15, [bias32] +IF X, paddd mx%2, m15 +IF Y, paddd my%2, m15 +IF Z, paddd mz%2, m15 +IF W, paddd mw%2, m15 + %endif + %ifnidn TYPE, SWS_PIXEL_F32 +IF X, vcvtdq2ps mx%2, mx%2 +IF Y, vcvtdq2ps my%2, my%2 +IF Z, vcvtdq2ps mz%2, mz%2 +IF W, vcvtdq2ps mw%2, mw%2 + %endif %endmacro -%macro filter_4x4_h 3 ; elems, type, sizeof_type -op filter_4x4_h%1_%2 +%macro READ_PLANAR_FH_4X4 1 +%xdefine FILTER_TYPE %1 %xdefine offset0q out0q %xdefine offset1q out1q %xdefine offset2q out2q @@ -738,39 +412,48 @@ op filter_4x4_h%1_%2 %xdefine weights tmp0q %xdefine offsets tmp2q %xdefine fltsize tmp1q - ; reserve some registers for the inner loops - push bxq - push offset0q - push offset1q - push offset2q - push offset3q - get_block_size - shl bxq, block_shift ; x := bx * block_size - mov weights, [implq + SwsOpImpl.priv] ; int16_t *weights - mov tmp1d, [implq + SwsOpImpl.priv + 8] ; size_t filter_size - mov offsets, [execq + SwsOpExec.in_offset_x] ; int32_t *offsets - lea offsets, [offsets + 4 * bxq] ; offsets += x * sizeof(int32_t) - imul bxq, fltsize - add weights, bxq ; weights += x * filter_size - shl fltsize, 2 ; fltsize *= 4 (number of pixels / iter) - filter_h8 %1, %2, %3, offsets - add weights, fltsize - filter_h8 %1, %2, %3, offsets + 32, 2 - mova m10, [scale_inv] - mulps mx, m10 -IF %1 > 1, mulps my, m10 -IF %1 > 2, mulps mz, m10 -IF %1 > 3, mulps mw, m10 - mulps mx2, m10 -IF %1 > 1, mulps my2, m10 -IF %1 > 2, mulps mz2, m10 -IF %1 > 3, mulps mw2, m10 - pop offset3q - pop offset2q - pop offset1q - pop offset0q - pop bxq - CONTINUE +%xdefine ILOAD_PX ILOAD_ %+ TYPE + +assert_idn FILTER_TYPE, SWS_PIXEL_F32 +%ifidn TYPE, SWS_PIXEL_F32 + %assign SIZEOF_WEIGHT 4 ; F32 +%else + %assign SIZEOF_WEIGHT 2 ; I16 +%endif + + ; reserve some registers for the inner loops + push bxq + push offset0q + push offset1q + push offset2q + push offset3q + shl_log2 bxq, FILTER_WIDTH ; x := bx * FILTER_WIDTH + mov weights, [implq + SwsOpImpl.priv] ; int16_t *weights + mov tmp1d, [implq + SwsOpImpl.priv + 8] ; size_t filter_size + mov offsets, [execq + SwsOpExec.in_offset_x] ; int32_t *offsets + lea offsets, [offsets + 4 * bxq] ; offsets += x * sizeof(int32_t) + imul bxq, fltsize + add weights, bxq ; weights += x * filter_size + shl fltsize, 2 ; fltsize *= 4 (number of pixels / iter) + filter_h8_4x4 offsets + add weights, fltsize + filter_h8_4x4 offsets + 32, 2 + VBROADCASTI128 m10, [scale_inv] + pop offset3q + pop offset2q + pop offset1q + pop offset0q + pop bxq + LOAD_CONT tmp0q +IF X, mulps mx, m10 +IF Y, mulps my, m10 +IF Z, mulps mz, m10 +IF W, mulps mw, m10 +IF X, mulps mx2, m10 +IF Y, mulps my2, m10 +IF Z, mulps mz2, m10 +IF W, mulps mw2, m10 + CONTINUE tmp0q %undef offset0q %undef offset1q %undef offset2q @@ -780,41 +463,238 @@ IF %1 > 3, mulps mw2, m10 %undef fltsize %endmacro -%macro generic_filter_fns 3 ; type, sizeof_type, sizeof_weight - filter_v 1, %1, %2, v - filter_v 2, %1, %2, v - filter_v 3, %1, %2, v - filter_v 4, %1, %2, v +;--------------------------------------------------------- +; Arithmetic operations - filter_v 1, %1, %2, fma_v - filter_v 2, %1, %2, fma_v - filter_v 3, %1, %2, fma_v - filter_v 4, %1, %2, fma_v - - filter_h 1, %1, %2, %3 - filter_h 2, %1, %2, %3 - filter_h 3, %1, %2, %3 - filter_h 4, %1, %2, %3 - - filter_4x4_h 1, %1, %2 - filter_4x4_h 2, %1, %2 - filter_4x4_h 3, %1, %2 - filter_4x4_h 4, %1, %2 +%macro SCALE 0 + vbroadcastss m8, [implq + SwsOpImpl.priv] + LOAD_CONT tmp0q +IF X, mulps mx, m8 +IF Y, mulps my, m8 +IF Z, mulps mz, m8 +IF W, mulps mw, m8 +IF X, mulps mx2, m8 +IF Y, mulps my2, m8 +IF Z, mulps mz2, m8 +IF W, mulps mw2, m8 + CONTINUE tmp0q %endmacro -%macro filter_fns 0 - generic_filter_fns U8, 1, 2 - generic_filter_fns U16, 2, 2 - generic_filter_fns F32, 4, 4 +%macro ADD 0 +IF X, vbroadcastss m8, [implq + SwsOpImpl.priv + 0 * BYTES] +IF Y, vbroadcastss m9, [implq + SwsOpImpl.priv + 1 * BYTES] +IF Z, vbroadcastss m10, [implq + SwsOpImpl.priv + 2 * BYTES] +IF W, vbroadcastss m11, [implq + SwsOpImpl.priv + 3 * BYTES] + LOAD_CONT tmp0q +IF X, addps mx, m8 +IF Y, addps my, m9 +IF Z, addps mz, m10 +IF W, addps mw, m11 +IF X, addps mx2, m8 +IF Y, addps my2, m9 +IF Z, addps mz2, m10 +IF W, addps mw2, m11 + CONTINUE tmp0q +%endmacro + +%macro MIN 0 +IF X, vbroadcastss m8, [implq + SwsOpImpl.priv + 0 * BYTES] +IF Y, vbroadcastss m9, [implq + SwsOpImpl.priv + 1 * BYTES] +IF Z, vbroadcastss m10, [implq + SwsOpImpl.priv + 2 * BYTES] +IF W, vbroadcastss m11, [implq + SwsOpImpl.priv + 3 * BYTES] + LOAD_CONT tmp0q +IF X, minps mx, m8 +IF Y, minps my, m9 +IF Z, minps mz, m10 +IF W, minps mw, m11 +IF X, minps mx2, m8 +IF Y, minps my2, m9 +IF Z, minps mz2, m10 +IF W, minps mw2, m11 + CONTINUE tmp0q +%endmacro + +%macro MAX 0 +IF X, vbroadcastss m8, [implq + SwsOpImpl.priv + 0 * BYTES] +IF Y, vbroadcastss m9, [implq + SwsOpImpl.priv + 1 * BYTES] +IF Z, vbroadcastss m10, [implq + SwsOpImpl.priv + 2 * BYTES] +IF W, vbroadcastss m11, [implq + SwsOpImpl.priv + 3 * BYTES] + LOAD_CONT tmp0q +IF X, maxps mx, m8 +IF Y, maxps my, m9 +IF Z, maxps mz, m10 +IF W, maxps mw, m11 +IF X, maxps mx2, m8 +IF Y, maxps my2, m9 +IF Z, maxps mz2, m10 +IF W, maxps mw2, m11 + CONTINUE tmp0q +%endmacro + +;--------------------------------------------------------- +; Linear operations + +%define LIN_MASK(I, J) (1 << (5 * (I) + (J))) + +%macro linear_muladd 5 ; dst, src, use_coef, coef, use_fma + %if INIT ; dst is already initialized + %if %3 && %5 + fmaddps %1, %4, %2, %1 + %elif %3 + mulps %4, %2 + addps %1, %4 + %else + addps %1, %2 + %endif + %else + %assign INIT 1 + %if %3 + mulps %1, %4, %2 + %else + mova %1, %2 + %endif + %endif +%endmacro + +%macro linear_row 3-4 ; dst, src, row, suffix +%xdefine NEED(J) (!(ZERO_MASK & LIN_MASK(%3, J))) +%xdefine LOAD(J) (NEED(J) && !(ONE_MASK & LIN_MASK(%3, J))) +%xdefine FMA(J) (EXACT_MASK & LIN_MASK(%3, J)) +%assign INIT 0 ; track whether `dst` already contains data + + %if !(ZERO_MASK & LIN_MASK(%3, 4)) ; nonzero output offset + %assign INIT 1 + vbroadcastss %1, [%2 + 4 * BYTES] + %endif +IF LOAD(0), vbroadcastss m12, [%2 + 0 * BYTES] +IF LOAD(1), vbroadcastss m13, [%2 + 1 * BYTES] +IF LOAD(2), vbroadcastss m14, [%2 + 2 * BYTES] +IF LOAD(3), vbroadcastss m15, [%2 + 3 * BYTES] +IF NEED(0), linear_muladd %1, mx%4, LOAD(0), m12, FMA(0) +IF NEED(1), linear_muladd %1, my%4, LOAD(1), m13, FMA(1) +IF NEED(2), linear_muladd %1, mz%4, LOAD(2), m14, FMA(2) +IF NEED(3), linear_muladd %1, mw%4, LOAD(3), m15, FMA(3) + assert INIT, SWS_UOP_LINEAR should not contain empty rows +%endmacro + +%macro LINEAR_FMA 3 +%assign ONE_MASK %1 +%assign ZERO_MASK %2 +%assign EXACT_MASK %3 + + mov tmp0q, [implq + SwsOpImpl.priv] ; address of matrix + LOAD_CONT tmp1q +IF1 X, linear_row m8, tmp0q + 0 * BYTES, 0 +IF1 Y, linear_row m9, tmp0q + 5 * BYTES, 1 +IF1 Z, linear_row m10, tmp0q + 10 * BYTES, 2 +IF1 W, linear_row m11, tmp0q + 15 * BYTES, 3 +IF X, mova mx, m8 +IF Y, mova my, m9 +IF Z, mova mz, m10 +IF W, mova mw, m11 +IF1 X, linear_row m8, tmp0q + 0 * BYTES, 0, 2 +IF1 Y, linear_row m9, tmp0q + 5 * BYTES, 1, 2 +IF1 Z, linear_row m10, tmp0q + 10 * BYTES, 2, 2 +IF1 W, linear_row m11, tmp0q + 15 * BYTES, 3, 2 +IF X, mova mx2, m8 +IF Y, mova my2, m9 +IF Z, mova mz2, m10 +IF W, mova mw2, m11 + CONTINUE tmp1q +%endmacro + +;--------------------------------------------------------- +; Dithering + +%macro VBROADCAST 3 ; size, dst, src + %if %1 == 4 + vbroadcastss %2, %3 + %elif %1 == 8 + vbroadcastsd %2, %3 + %elif %1 == 16 + VBROADCASTF128 %2, %3 + %else + mova %2, %3 + %endif +%endmacro + +%macro DITHER 5 ; x_off, y_off, z_off, w_off, size_log2 +%assign SIZE_LOG2 %5 +%assign SIZE (1 << SIZE_LOG2) +%assign STRIDE (SIZE * BYTES) + ; dither matrix is stored indirectly at the private data address + mov tmp1q, [implq + SwsOpImpl.priv] + ; add y offset. note that for 2x2, we would only need to look at the + ; sign of `y`, but this special case is ignored for simplicity reasons + ; (and because the current upstream format code never generates matrices + ; that small) + mov tmp0d, yd + and tmp0d, SIZE - 1 + shl_log2 tmp0d, STRIDE + add tmp1q, tmp0q + %if STRIDE > BLOCK_SIZE + ; need to add in x offset + mov tmp0d, bxd + shl tmp0d, BLOCK_SIZE + and tmp0d, STRIDE - 1 + add tmp1q, tmp0q + %endif + %xdefine dither_addr_x (tmp1q + %1 * STRIDE) + %xdefine dither_addr_y (tmp1q + %2 * STRIDE) + %xdefine dither_addr_z (tmp1q + %3 * STRIDE) + %xdefine dither_addr_w (tmp1q + %4 * STRIDE) + %if STRIDE > mmsize + LOAD_CONT tmp0q +IF X, addps mx, [dither_addr_x] +IF Y, addps my, [dither_addr_y] +IF Z, addps mz, [dither_addr_z] +IF W, addps mw, [dither_addr_w] +IF X, addps mx2, [dither_addr_x + mmsize] +IF Y, addps my2, [dither_addr_y + mmsize] +IF Z, addps mz2, [dither_addr_z + mmsize] +IF W, addps mw2, [dither_addr_w + mmsize] + %else +IF X, VBROADCAST STRIDE, m8, [dither_addr_x] +IF Y, VBROADCAST STRIDE, m9, [dither_addr_y] +IF Z, VBROADCAST STRIDE, m10, [dither_addr_z] +IF W, VBROADCAST STRIDE, m11, [dither_addr_w] + LOAD_CONT tmp0q +IF X, addps mx, m8 +IF Y, addps my, m9 +IF Z, addps mz, m10 +IF W, addps mw, m11 +IF X, addps mx2, m8 +IF Y, addps my2, m9 +IF Z, addps mz2, m10 +IF W, addps mw2, m11 + %endif + CONTINUE tmp0q +%endmacro + +;--------------------------------------------------------- +; Instantiate above macros to generate all uop kernels + +%macro decl_filter_ops 1 ; type + decl_suffix _4x4, DECL_%1_READ_PLANAR_FH (READ_PLANAR_FH_4X4) + + DECL_%1_READ_PLANAR_FH (READ_PLANAR_FH) + DECL_%1_READ_PLANAR_FV (READ_PLANAR_FV) + DECL_%1_READ_PLANAR_FV_FMA (READ_PLANAR_FV_FMA) +%endmacro + +%macro decl_float_ops 1 ; type + DECL_%1_SCALE (SCALE) + DECL_%1_ADD (ADD) + DECL_%1_MIN (MIN) + DECL_%1_MAX (MAX) + DECL_%1_LINEAR_FMA (LINEAR_FMA) + DECL_%1_DITHER (DITHER) %endmacro INIT_YMM avx2 -decl_common_patterns conv8to32f -decl_common_patterns conv16to32f -decl_common_patterns conv32fto8 -decl_common_patterns conv32fto16 -decl_common_patterns min_max -decl_common_patterns scale -dither_fns -linear_fns -filter_fns +%assign V2 1 + +decl_filter_ops U8 +decl_filter_ops U16 +decl_filter_ops F32 +decl_float_ops F32 diff --git a/libswscale/x86/ops_include.asm b/libswscale/x86/ops_include.asm index dcbf5d8eaa..cc44a247b0 100644 --- a/libswscale/x86/ops_include.asm +++ b/libswscale/x86/ops_include.asm @@ -70,9 +70,9 @@ ; if we load 8-bit values and convert them to 32-bit floats internally, then ; we would have an operation chain which combines an SSE4 V2=0 u8 kernel (128 ; bits = 16 pixels) with an AVX2 V2=1 f32 kernel (512 bits = 16 pixels). This -; keeps the number of pixels being processed (the block size) constant. The -; V2 setting is suffixed to the operation name (_m1 or _m2) during name -; mangling. +; keeps the number of pixels being processed (BLOCK_WIDTH) constant. The V2 +; setting is suffixed to the operation name (_m1 or _m2), along with any other +; custom NAME_SUFFIX (e.g. for operation variants). ; ; This design leaves us with the following set of possibilities: ; @@ -92,15 +92,7 @@ ; - 16-bit kernels: m1_avx2, m2_avx2 ; - 32-bit kernels: m2_avx2 ; -; This is achieved by macro'ing each operation kernel and declaring it once -; per SIMD version, and (if needed) once per V2 setting using decl_v2. (See -; the bottom of ops_int.asm for an example) -; -; Finally, we overload some operation kernel to different number of components, -; using the `decl_pattern` and `decl_common_patterns` macros. Inside these -; kernels, the variables X, Y, Z and W will be set to 0 or 1 respectively, -; depending on which components are active for this particular kernel instance. -; They will receive the _pXYZW prefix during name mangling. +; See the bottom of ops_int.asm for an example. struc SwsOpExec .in0 resq 1 @@ -147,17 +139,55 @@ struc SwsOpImpl .next resb 0 endstruc +%define SWS_COMP_NONE 0 +%define SWS_COMP_ALL 0xF +%define SWS_COMP(X) (1 << (X)) +%define SWS_COMP_TEST(mask, X) (((mask) >> X) & 1) +%define SWS_COMP_INV(mask) ((mask) ^ SWS_COMP_ALL) +%define SWS_COMP_ELEMS(N) ((1 << (N)) - 1) + ;--------------------------------------------------------- ; Common macros for declaring operations -; Declare an operation kernel with the correct name mangling. -%macro op 1 ; name - %ifdef X - %define ADD_PAT(name) p %+ X %+ Y %+ Z %+ W %+ _ %+ name +; Declare an operation kernel, calling a provided macro to generate the body. +; Note: This is automatically called by the DECL_*() macros +%macro DECL_OP 5-6+ ; macro, name, type, uop, mask + ; Declare named variables for common macro parameters + %ifdef NAME_SUFFIX + %xdefine NAME %2 %+ NAME_SUFFIX %else - %define ADD_PAT(name) name + %xdefine NAME %2 %endif + %xdefine TYPE %3 + %xdefine UOP %4 + %xdefine MASK %5 + ; Declare X/Y/Z/W based on the provided mask + %assign X SWS_COMP_TEST(MASK, 0) + %assign Y SWS_COMP_TEST(MASK, 1) + %assign Z SWS_COMP_TEST(MASK, 2) + %assign W SWS_COMP_TEST(MASK, 3) + %assign COMPS (X + Y + Z + W) + + ; Declare BYTES and BITS based on the ops type + %ifidn TYPE, SWS_PIXEL_U8 + %assign BYTES 1 + %elifidn TYPE, SWS_PIXEL_U16 + %assign BYTES 2 + %else + %assign BYTES 4 + %endif + %assign BITS (BYTES * 8) + + ; Calculate block size helpers + %ifdef V2 + %assign BLOCK_SIZE (mmsize * (1 + V2)) + %else + %assign BLOCK_SIZE (mmsize) + %endif + %assign BLOCK_WIDTH (BLOCK_SIZE / BYTES) + + ; Add the correct name mangling / suffix for decl_v2 %ifdef V2 %if V2 %define ADD_MUL(name) name %+ _m2 @@ -168,12 +198,32 @@ endstruc %define ADD_MUL(name) name %endif - cglobal ADD_PAT(ADD_MUL(%1)), 0, 0, 0 ; already allocated by entry point + cglobal ADD_MUL(NAME), 0, 0, 0 ; already allocated by entry point + %1 %6 ; call the provided macro to generate the kernel body - %undef ADD_PAT + %undef NAME + %undef UOP + %undef TYPE + %undef MASK + %undef X + %undef Y + %undef Z + %undef W + %undef COMPS + %undef BYTES + %undef BITS + %undef BLOCK_SIZE + %undef BLOCK_WIDTH %undef ADD_MUL %endmacro +; Declare a set of operations with a custom name suffix +%macro decl_suffix 2+ ; suffix, func + %xdefine NAME_SUFFIX %1 + %2 + %undef NAME_SUFFIX +%endmacro + ; Declare a set of operations with the high half enabled / disabled %macro decl_v2 2+ ; v2, func %assign V2 %1 @@ -181,27 +231,6 @@ endstruc %undef V2 %endmacro -; Declare an operation kernel specialized to a given subset of active components -%macro decl_pattern 5+ ; X, Y, Z, W, func - %assign X %1 - %assign Y %2 - %assign Z %3 - %assign W %4 - %5 - %undef X - %undef Y - %undef Z - %undef W -%endmacro - -; Declare an operation kernel specialized to each common component pattern -%macro decl_common_patterns 1+ ; func - decl_pattern 1, 0, 0, 0, %1 ; y - decl_pattern 1, 0, 0, 1, %1 ; ya - decl_pattern 1, 1, 1, 0, %1 ; yuv - decl_pattern 1, 1, 1, 1, %1 ; yuva -%endmacro - ;--------------------------------------------------------- ; Common names for the internal calling convention %define mx m0 @@ -287,6 +316,9 @@ endstruc CONTINUE tmp0q %endmacro +;--------------------------------------------------------- +; Miscellaneous helpers + ; Helper for inline conditionals; used to conditionally include single lines %macro IF 2+ ; cond, body %if %1 @@ -300,3 +332,43 @@ endstruc %2 %endif %endmacro + +%macro shl_log2 2 ; dst, amount + %if %2 == 64 + shl %1, 6 + %elif %2 == 32 + shl %1, 5 + %elif %2 == 16 + shl %1, 4 + %elif %2 == 8 + shl %1, 3 + %elif %2 == 4 + shl %1, 2 + %elif %2 == 2 + shl %1, 1 + %elif %2 == 1 + ; no-op + %else + %error "Unsupported value for shl_log2" + %endif +%endmacro + +%macro assert 1-2 ; cond, message + %if !(%1) + %if %0 > 1 + %error %2 + %else + %error Assertion failed: %1 + %endif + %endif +%endmacro + +%macro assert_idn 2-3 ; expr1, expr2, message + %ifnidn %1, %2 + %if %0 > 2 + %error %3 + %else + %error Assertion failed: %1 == %2 + %endif + %endif +%endmacro diff --git a/libswscale/x86/ops_int.asm b/libswscale/x86/ops_int.asm index 7c366a8f8d..1779b15faa 100644 --- a/libswscale/x86/ops_int.asm +++ b/libswscale/x86/ops_int.asm @@ -1,5 +1,5 @@ ;****************************************************************************** -;* Copyright (c) 2025 Niklas Haas +;* Copyright (c) 2025-2026 Niklas Haas ;* ;* This file is part of FFmpeg. ;* @@ -23,9 +23,6 @@ SECTION_RODATA align 16 -expand16_shuf: db 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14 -expand32_shuf: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 - read8_unpack2: db 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15 read8_unpack3: db 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, -1, -1, -1, -1 read8_unpack4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 @@ -35,255 +32,117 @@ read16_unpack4: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, write8_pack2: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 write8_pack3: db 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, -1, -1, -1, -1 write16_pack3: db 0, 1, 4, 5, 8, 9, 2, 3, 6, 7, 10, 11, -1, -1, -1, -1 - %define write8_pack4 read8_unpack4 %define write16_pack4 read16_unpack2 %define write16_pack2 read16_unpack4 -align 32 -bits_shuf: db 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, \ - 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 bits_mask: db 128, 64, 32, 16, 8, 4, 2, 1,128, 64, 32, 16, 8, 4, 2, 1 -bits_reverse: db 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, +bits_reverse: db 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 +nibbles_pack: times 8 dw 0x0110 + +swap16: db 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 +swap32: db 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 +expand16: db 0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14 +expand32: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 align 32 +bits_shuf: times 8 db 0 + times 8 db 1 + times 8 db 2 + times 8 db 3 + mask1: times 32 db 0x01 mask2: times 32 db 0x03 mask3: times 32 db 0x07 mask4: times 32 db 0x0F -const1b equ mask1 -const1w: times 16 dw 0x01 +const1b equ mask1 +const1w: times 16 dw 0x1 SECTION .text +;--------------------------------------------------------- +; Helper macros for BITS-dependent sized instructions + +%macro bitfn 2+ + %if BITS == 8 + %1%+b %2 + %elif BITS == 16 + %1%+w %2 + %elif BITS == 32 + %1%+d %2 + %endif +%endmacro + +%define psllx bitfn psll, +%define psrlx bitfn psrl, +%define pcmpeqx bitfn pcmpeq, +%define pmullx bitfn pmull, +%define vpbroadcastx bitfn vpbroadcast, + ;--------------------------------------------------------- ; Planar reads / writes -%macro read_planar 1 ; elems -op read_planar%1 - movu mx, [in0q] -IF %1 > 1, movu my, [in1q] -IF %1 > 2, movu mz, [in2q] -IF %1 > 3, movu mw, [in3q] +%macro READ_PLANAR 0 +IF X, movu mx, [in0q] +IF Y, movu my, [in1q] +IF Z, movu mz, [in2q] +IF W, movu mw, [in3q] %if V2 - movu mx2, [in0q + mmsize] -IF %1 > 1, movu my2, [in1q + mmsize] -IF %1 > 2, movu mz2, [in2q + mmsize] -IF %1 > 3, movu mw2, [in3q + mmsize] +IF X, movu mx2, [in0q + mmsize] +IF Y, movu my2, [in1q + mmsize] +IF Z, movu mz2, [in2q + mmsize] +IF W, movu mw2, [in3q + mmsize] %endif - LOAD_CONT tmp0q - add in0q, mmsize * (1 + V2) -IF %1 > 1, add in1q, mmsize * (1 + V2) -IF %1 > 2, add in2q, mmsize * (1 + V2) -IF %1 > 3, add in3q, mmsize * (1 + V2) - CONTINUE tmp0q -%endmacro - -%macro write_planar 1 ; elems -op write_planar%1 - movu [out0q], mx -IF %1 > 1, movu [out1q], my -IF %1 > 2, movu [out2q], mz -IF %1 > 3, movu [out3q], mw -%if V2 - movu [out0q + mmsize], mx2 -IF %1 > 1, movu [out1q + mmsize], my2 -IF %1 > 2, movu [out2q + mmsize], mz2 -IF %1 > 3, movu [out3q + mmsize], mw2 -%endif - add out0q, mmsize * (1 + V2) -IF %1 > 1, add out1q, mmsize * (1 + V2) -IF %1 > 2, add out2q, mmsize * (1 + V2) -IF %1 > 3, add out3q, mmsize * (1 + V2) - RET -%endmacro - -%macro read_packed2 1 ; depth -op read%1_packed2 - movu m8, [in0q + 0*mmsize] - movu m9, [in0q + 1*mmsize] - IF V2, movu m10, [in0q + 2*mmsize] - IF V2, movu m11, [in0q + 3*mmsize] -IF %1 < 32, VBROADCASTI128 m12, [read%1_unpack2] - LOAD_CONT tmp0q - add in0q, mmsize * (2 + V2 * 2) -%if %1 == 32 - shufps m8, m8, q3120 - shufps m9, m9, q3120 - IF V2, shufps m10, m10, q3120 - IF V2, shufps m11, m11, q3120 -%else - pshufb m8, m12 ; { X0 Y0 | X1 Y1 } - pshufb m9, m12 ; { X2 Y2 | X3 Y3 } - IF V2, pshufb m10, m12 - IF V2, pshufb m11, m12 -%endif - unpcklpd mx, m8, m9 ; { X0 X2 | X1 X3 } - unpckhpd my, m8, m9 ; { Y0 Y2 | Y1 Y3 } - IF V2, unpcklpd mx2, m10, m11 - IF V2, unpckhpd my2, m10, m11 -%if cpuflag(avx2) - vpermq mx, mx, q3120 ; { X0 X1 | X2 X3 } - vpermq my, my, q3120 ; { Y0 Y1 | Y2 Y3 } - IF V2, vpermq mx2, mx2, q3120 - IF V2, vpermq my2, my2, q3120 -%endif - CONTINUE tmp0q -%endmacro - -%macro write_packed2 1 ; depth -op write%1_packed2 -IF %1 < 32, VBROADCASTI128 m12, [write%1_pack2] -%if cpuflag(avx2) - vpermq mx, mx, q3120 ; { X0 X2 | X1 X3 } - vpermq my, my, q3120 ; { Y0 Y2 | Y1 Y3 } - IF V2, vpermq mx2, mx2, q3120 - IF V2, vpermq my2, my2, q3120 -%endif - unpcklpd m8, mx, my ; { X0 Y0 | X1 Y1 } - unpckhpd m9, mx, my ; { X2 Y2 | X3 Y3 } - IF V2, unpcklpd m10, mx2, my2 - IF V2, unpckhpd m11, mx2, my2 -%if %1 == 32 - shufps m8, m8, q3120 - shufps m9, m9, q3120 - IF V2, shufps m10, m10, q3120 - IF V2, shufps m11, m11, q3120 -%else - pshufb m8, m12 - pshufb m9, m12 - IF V2, pshufb m10, m12 - IF V2, pshufb m11, m12 -%endif - movu [out0q + 0*mmsize], m8 - movu [out0q + 1*mmsize], m9 -IF V2, movu [out0q + 2*mmsize], m10 -IF V2, movu [out0q + 3*mmsize], m11 - add out0q, mmsize * (2 + V2 * 2) - RET -%endmacro - -; helper macro reused for both 3 and 4 component packed reads -%macro read_packed_inner 7 ; x, y, z, w, addr, num, depth - movu xm8, [%5 + 0 * %6] - movu xm9, [%5 + 4 * %6] - movu xm10, [%5 + 8 * %6] - movu xm11, [%5 + 12 * %6] - %if cpuflag(avx2) - vinserti128 m8, m8, [%5 + 16 * %6], 1 - vinserti128 m9, m9, [%5 + 20 * %6], 1 - vinserti128 m10, m10, [%5 + 24 * %6], 1 - vinserti128 m11, m11, [%5 + 28 * %6], 1 - %endif - %if %7 == 32 - mova %1, m8 - mova %2, m9 - mova %3, m10 - mova %4, m11 - %else - pshufb %1, m8, m12 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 } - pshufb %2, m9, m12 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 } - pshufb %3, m10, m12 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 } - pshufb %4, m11, m12 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 } - %endif - punpckldq m8, %1, %2 ; { X0 X1 Y0 Y1 | X4 X5 Y4 Y5 } - punpckldq m9, %3, %4 ; { X2 X3 Y2 Y3 | X6 X7 Y6 Y7 } - punpckhdq m10, %1, %2 ; { Z0 Z1 W0 W1 | Z4 Z5 W4 W5 } - punpckhdq m11, %3, %4 ; { Z2 Z3 W2 W3 | Z6 Z7 W6 W7 } - punpcklqdq %1, m8, m9 ; { X0 X1 X2 X3 | X4 X5 X6 X7 } - punpckhqdq %2, m8, m9 ; { Y0 Y1 Y2 Y3 | Y4 Y5 Y6 Y7 } - punpcklqdq %3, m10, m11 ; { Z0 Z1 Z2 Z3 | Z4 Z5 Z6 Z7 } -IF %6 > 3, punpckhqdq %4, m10, m11 ; { W0 W1 W2 W3 | W4 W5 W6 W7 } -%endmacro - -%macro read_packed 2 ; num, depth -op read%2_packed%1 -IF %2 < 32, VBROADCASTI128 m12, [read%2_unpack%1] - LOAD_CONT tmp0q - read_packed_inner mx, my, mz, mw, in0q, %1, %2 -IF1 V2, read_packed_inner mx2, my2, mz2, mw2, in0q + %1 * mmsize, %1, %2 - add in0q, %1 * mmsize * (1 + V2) - CONTINUE tmp0q -%endmacro - -%macro write_packed_inner 7 ; x, y, z, w, addr, num, depth - punpckldq m8, %1, %2 ; { X0 Y0 X1 Y1 | X4 Y4 X5 Y5 } - punpckldq m9, %3, %4 ; { Z0 W0 Z1 W1 | Z4 W4 Z5 W5 } - punpckhdq m10, %1, %2 ; { X2 Y2 X3 Y3 | X6 Y6 X7 Y7 } - punpckhdq m11, %3, %4 ; { Z2 W2 Z3 W3 | Z6 W6 Z7 W7 } - punpcklqdq %1, m8, m9 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 } - punpckhqdq %2, m8, m9 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 } - punpcklqdq %3, m10, m11 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 } - punpckhqdq %4, m10, m11 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 } - %if %7 == 32 - mova m8, %1 - mova m9, %2 - mova m10, %3 - mova m11, %4 - %else - pshufb m8, %1, m12 - pshufb m9, %2, m12 - pshufb m10, %3, m12 - pshufb m11, %4, m12 - %endif - movu [%5 + 0*%6], xm8 - movu [%5 + 4*%6], xm9 - movu [%5 + 8*%6], xm10 - movu [%5 + 12*%6], xm11 - %if cpuflag(avx2) - vextracti128 [%5 + 16*%6], m8, 1 - vextracti128 [%5 + 20*%6], m9, 1 - vextracti128 [%5 + 24*%6], m10, 1 - vextracti128 [%5 + 28*%6], m11, 1 - %endif -%endmacro - -%macro write_packed 2 ; num, depth -op write%2_packed%1 -IF %2 < 32, VBROADCASTI128 m12, [write%2_pack%1] - write_packed_inner mx, my, mz, mw, out0q, %1, %2 -IF1 V2, write_packed_inner mx2, my2, mz2, mw2, out0q + %1 * mmsize, %1, %2 - add out0q, %1 * mmsize * (1 + V2) - RET -%endmacro - -%macro rw_packed 1 ; depth - read_packed2 %1 - read_packed 3, %1 - read_packed 4, %1 - write_packed2 %1 - write_packed 3, %1 - write_packed 4, %1 -%endmacro - -%macro read_nibbles 0 -op read_nibbles1 -%if cpuflag(avx2) - movu xmx, [in0q] -IF V2, movu xmx2, [in0q + 16] -%else - movq xmx, [in0q] -IF V2, movq xmx2, [in0q + 8] -%endif - VBROADCASTI128 m8, [mask4] LOAD_CONT tmp0q - add in0q, (mmsize >> 1) * (1 + V2) - pmovzxbw mx, xmx -IF V2, pmovzxbw mx2, xmx2 - psllw my, mx, 8 -IF V2, psllw my2, mx2, 8 - psrlw mx, 4 -IF V2, psrlw mx2, 4 - pand my, m8 -IF V2, pand my2, m8 - por mx, my -IF V2, por mx2, my2 +IF X, add in0q, BLOCK_SIZE +IF Y, add in1q, BLOCK_SIZE +IF Z, add in2q, BLOCK_SIZE +IF W, add in3q, BLOCK_SIZE CONTINUE tmp0q %endmacro -%macro read_bits 0 -op read_bits1 +%macro READ_NIBBLE 0 +assert COMPS == 1 + VBROADCASTI128 m12, [mask4] + LOAD_CONT tmp0q + pmovzxbw mx, [in0q] +IF V2, pmovzxbw mx2, [in0q + (mmsize >> 1)] + add in0q, BLOCK_WIDTH >> 1 + psllw m8, mx, 8 ; { 00 AB 00 CD ... } + psrlw mx, 4 ; { 0A 00 0C 00 ... } + pand m8, m12 ; { 00 0B 00 0D ... } + por mx, m8 ; { 0A 0B 0C 0D ... } +%if V2 + psllw m9, mx2, 8 + psrlw mx2, 4 + pand m9, m12 + por mx2, m9 +%endif + CONTINUE tmp0q +%endmacro + +%macro WRITE_NIBBLE 0 +assert COMPS == 1 + VBROADCASTI128 m12, [nibbles_pack] + pmaddubsw mx, m12 + packuswb mx, mx +%if V2 + pmaddubsw mx2, m12 + packuswb mx2, mx2 +%endif +%if cpuflag(avx2) + vpermq mx, mx, q3120 +IF V2, vpermq mx2, mx2, q3120 +%endif + movu [out0q], xmx +IF V2, movu [out0q + (mmsize >> 1)], xmx2 + add out0q, BLOCK_WIDTH >> 1 + RET +%endmacro + +%macro READ_BIT 0 +assert COMPS == 1 %if cpuflag(avx2) vpbroadcastd mx, [in0q] IF V2, vpbroadcastd mx2, [in0q + 4] @@ -295,7 +154,7 @@ IF V2, movd mx2, [in0q + 2] VBROADCASTI128 m9, [bits_mask] VBROADCASTI128 m10, [const1b] LOAD_CONT tmp0q - add in0q, (mmsize >> 3) * (1 + V2) + add in0q, BLOCK_WIDTH >> 3 pshufb mx, m8 IF V2, pshufb mx2, m8 pand mx, m9 @@ -307,10 +166,8 @@ IF V2, pand mx2, m10 CONTINUE tmp0q %endmacro -; TODO: write_nibbles - -%macro write_bits 0 -op write_bits1 +%macro WRITE_BIT 0 +assert COMPS == 1 VBROADCASTI128 m8, [bits_reverse] psllw mx, 7 IF V2, psllw mx2, 7 @@ -325,57 +182,204 @@ IF V2, mov [out0q + (mmsize >> 3)], tmp1d mov [out0q], tmp0w IF V2, mov [out0q + (mmsize >> 3)], tmp1w %endif - add out0q, (mmsize >> 3) * (1 + V2) + add out0q, BLOCK_WIDTH >> 3 RET %endmacro -;-------------------------- -; Pixel packing / unpacking +%macro WRITE_PLANAR 0 +IF X, movu [out0q], mx +IF Y, movu [out1q], my +IF Z, movu [out2q], mz +IF W, movu [out3q], mw +%if V2 +IF X, movu [out0q + mmsize], mx2 +IF Y, movu [out1q + mmsize], my2 +IF Z, movu [out2q + mmsize], mz2 +IF W, movu [out3q + mmsize], mw2 +%endif +IF X, add out0q, BLOCK_WIDTH +IF Y, add out1q, BLOCK_WIDTH +IF Z, add out2q, BLOCK_WIDTH +IF W, add out3q, BLOCK_WIDTH + RET +%endmacro -%macro pack_generic 3-4 0 ; x, y, z, w -op pack_%1%2%3%4 +;--------------------------------------------------------- +; Packed reads, packing and unpacking + +%macro read_packed2 0 + movu m8, [in0q + 0 * mmsize] + movu m9, [in0q + 1 * mmsize] +IF V2, movu m10, [in0q + 2 * mmsize] +IF V2, movu m11, [in0q + 3 * mmsize] + %if BITS == 32 + shufps m8, m8, q3120 + shufps m9, m9, q3120 +IF V2, shufps m10, m10, q3120 +IF V2, shufps m11, m11, q3120 + %else + pshufb m8, m12 ; { X0 Y0 | X1 Y1 } + pshufb m9, m12 ; { X2 Y2 | X3 Y3 } +IF V2, pshufb m10, m12 +IF V2, pshufb m11, m12 + %endif + unpcklpd mx, m8, m9 ; { X0 X2 | X1 X3 } + unpckhpd my, m8, m9 ; { Y0 Y2 | Y1 Y3 } +IF V2, unpcklpd mx2, m10, m11 +IF V2, unpckhpd my2, m10, m11 + %if cpuflag(avx2) + vpermq mx, mx, q3120 ; { X0 X1 | X2 X3 } + vpermq my, my, q3120 ; { Y0 Y1 | Y2 Y3 } +IF V2, vpermq mx2, mx2, q3120 +IF V2, vpermq my2, my2, q3120 + %endif +%endmacro + +%macro read_packed34 5 ; x, y, z, w, addr + movu xm8, [%5 + 0 * COMPS] + movu xm9, [%5 + 4 * COMPS] + movu xm10, [%5 + 8 * COMPS] + movu xm11, [%5 + 12 * COMPS] + %if cpuflag(avx2) + vinserti128 m8, m8, [%5 + 16 * COMPS], 1 + vinserti128 m9, m9, [%5 + 20 * COMPS], 1 + vinserti128 m10, m10, [%5 + 24 * COMPS], 1 + vinserti128 m11, m11, [%5 + 28 * COMPS], 1 + %endif + %if BITS == 32 + mova %1, m8 + mova %2, m9 + mova %3, m10 + mova %4, m11 + %else + pshufb %1, m8, m12 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 } + pshufb %2, m9, m12 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 } + pshufb %3, m10, m12 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 } + pshufb %4, m11, m12 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 } + %endif + punpckldq m8, %1, %2 ; { X0 X1 Y0 Y1 | X4 X5 Y4 Y5 } + punpckldq m9, %3, %4 ; { X2 X3 Y2 Y3 | X6 X7 Y6 Y7 } + punpckhdq m10, %1, %2 ; { Z0 Z1 W0 W1 | Z4 Z5 W4 W5 } + punpckhdq m11, %3, %4 ; { Z2 Z3 W2 W3 | Z6 Z7 W6 W7 } + punpcklqdq %1, m8, m9 ; { X0 X1 X2 X3 | X4 X5 X6 X7 } + punpckhqdq %2, m8, m9 ; { Y0 Y1 Y2 Y3 | Y4 Y5 Y6 Y7 } + punpcklqdq %3, m10, m11 ; { Z0 Z1 Z2 Z3 | Z4 Z5 Z6 Z7 } +IF W, punpckhqdq %4, m10, m11 ; { W0 W1 W2 W3 | W4 W5 W6 W7 } +%endmacro + +%macro READ_PACKED 0 + %if BITS < 32 + VBROADCASTI128 m12, [ read %+ BITS %+ _unpack %+ COMPS ] + %endif + LOAD_CONT tmp0q + %if COMPS == 2 + read_packed2 + %else + read_packed34 mx, my, mz, mw, in0q +IF1 V2, read_packed34 mx2, my2, mz2, mw2, in0q + mmsize * COMPS + %endif + add in0q, BLOCK_SIZE * COMPS + CONTINUE tmp0q +%endmacro + +%macro write_packed2 0 + %if cpuflag(avx2) + vpermq mx, mx, q3120 ; { X0 X2 | X1 X3 } + vpermq my, my, q3120 ; { Y0 Y2 | Y1 Y3 } +IF V2, vpermq mx2, mx2, q3120 +IF V2, vpermq my2, my2, q3120 + %endif + unpcklpd m8, mx, my ; { X0 Y0 | X1 Y1 } + unpckhpd m9, mx, my ; { X2 Y2 | X3 Y3 } +IF V2, unpcklpd m10, mx2, my2 +IF V2, unpckhpd m11, mx2, my2 + %if BITS == 32 + shufps m8, m8, q3120 + shufps m9, m9, q3120 +IF V2, shufps m10, m10, q3120 +IF V2, shufps m11, m11, q3120 + %else + pshufb m8, m12 + pshufb m9, m12 +IF V2, pshufb m10, m12 +IF V2, pshufb m11, m12 + %endif + movu [out0q + 0 * mmsize], m8 + movu [out0q + 1 * mmsize], m9 +IF V2, movu [out0q + 2 * mmsize], m10 +IF V2, movu [out0q + 3 * mmsize], m11 +%endmacro + +%macro write_packed34 5 ; x, y, z, w, addr + punpckldq m8, %1, %2 ; { X0 Y0 X1 Y1 | X4 Y4 X5 Y5 } + punpckldq m9, %3, %4 ; { Z0 W0 Z1 W1 | Z4 W4 Z5 W5 } + punpckhdq m10, %1, %2 ; { X2 Y2 X3 Y3 | X6 Y6 X7 Y7 } + punpckhdq m11, %3, %4 ; { Z2 W2 Z3 W3 | Z6 W6 Z7 W7 } + punpcklqdq %1, m8, m9 ; { X0 Y0 Z0 W0 | X4 Y4 Z4 W4 } + punpckhqdq %2, m8, m9 ; { X1 Y1 Z1 W1 | X5 Y5 Z5 W5 } + punpcklqdq %3, m10, m11 ; { X2 Y2 Z2 W2 | X6 Y6 Z6 W6 } + punpckhqdq %4, m10, m11 ; { X3 Y3 Z3 W3 | X7 Y7 Z7 W7 } + %if BITS == 32 + mova m8, %1 + mova m9, %2 + mova m10, %3 + mova m11, %4 + %else + pshufb m8, %1, m12 + pshufb m9, %2, m12 + pshufb m10, %3, m12 + pshufb m11, %4, m12 + %endif + movu [%5 + 0 * COMPS], xm8 + movu [%5 + 4 * COMPS], xm9 + movu [%5 + 8 * COMPS], xm10 + movu [%5 + 12 * COMPS], xm11 + %if cpuflag(avx2) + vextracti128 [%5 + 16 * COMPS], m8, 1 + vextracti128 [%5 + 20 * COMPS], m9, 1 + vextracti128 [%5 + 24 * COMPS], m10, 1 + vextracti128 [%5 + 28 * COMPS], m11, 1 + %endif +%endmacro + +%macro WRITE_PACKED 0 + %if BITS < 32 + VBROADCASTI128 m12, [ write %+ BITS %+ _pack %+ COMPS ] + %endif + LOAD_CONT tmp0q + %if COMPS == 2 + write_packed2 + %else + write_packed34 mx, my, mz, mw, out0q +IF1 V2, write_packed34 mx2, my2, mz2, mw2, out0q + mmsize * COMPS + %endif + add out0q, BLOCK_SIZE * COMPS + RET +%endmacro + +%macro PACK 4 ; x, y, z, w ; pslld works for all sizes because the input should not overflow IF %2, pslld mx, %4+%3+%2 IF %3, pslld my, %4+%3 IF %4, pslld mz, %4 -IF %2, por mx, my -IF %3, por mx, mz -IF %4, por mx, mw +IF Y, por mx, my +IF Z, por mx, mz +IF W, por mx, mw %if V2 IF %2, pslld mx2, %4+%3+%2 IF %3, pslld my2, %4+%3 IF %4, pslld mz2, %4 -IF %2, por mx2, my2 -IF %3, por mx2, mz2 -IF %4, por mx2, mw2 +IF Y, por mx2, my2 +IF Z, por mx2, mz2 +IF W, por mx2, mw2 %endif CONTINUE %endmacro -%macro unpack 5-6 0 ; type, bits, x, y, z, w -op unpack_%3%4%5%6 - ; clear high bits by shifting left -IF %6, vpsll%1 mw, mx, %2 - (%6) -IF %5, vpsll%1 mz, mx, %2 - (%6+%5) -IF %4, vpsll%1 my, mx, %2 - (%6+%5+%4) - psrl%1 mx, %4+%5+%6 -IF %4, psrl%1 my, %2 - %4 -IF %5, psrl%1 mz, %2 - %5 -IF %6, psrl%1 mw, %2 - %6 - %if V2 -IF %6, vpsll%1 mw2, mx2, %2 - (%6) -IF %5, vpsll%1 mz2, mx2, %2 - (%6+%5) -IF %4, vpsll%1 my2, mx2, %2 - (%6+%5+%4) - psrl%1 mx2, %4+%5+%6 -IF %4, psrl%1 my2, %2 - %4 -IF %5, psrl%1 mz2, %2 - %5 -IF %6, psrl%1 mw2, %2 - %6 - %endif - CONTINUE -%endmacro - -%macro unpack8 3 ; x, y, z -op unpack_%1%2%3 %+ 0 +%macro UNPACK 4 ; x, y, z, w + LOAD_CONT tmp0q +%if BITS == 8 + assert MASK == SWS_COMP_ELEMS(3) pand mz, mx, [mask%3] psrld my, mx, %3 psrld mx, %3+%2 @@ -388,522 +392,464 @@ op unpack_%1%2%3 %+ 0 pand my2, [mask%2] pand mx2, [mask%1] %endif - CONTINUE -%endmacro - -;--------------------------------------------------------- -; Generic byte order shuffle (packed swizzle, endian, etc) - -%macro shuffle 0 -op shuffle - VBROADCASTI128 m8, [implq + SwsOpImpl.priv] - LOAD_CONT tmp0q -IF X, pshufb mx, m8 -IF Y, pshufb my, m8 -IF Z, pshufb mz, m8 -IF W, pshufb mw, m8 -%if V2 -IF X, pshufb mx2, m8 -IF Y, pshufb my2, m8 -IF Z, pshufb mz2, m8 -IF W, pshufb mw2, m8 +%else + ; clear high bits by shifting left +IF W, psllx mw, mx, BITS - (%4) +IF Z, psllx mz, mx, BITS - (%4+%3) +IF Y, psllx my, mx, BITS - (%4+%3+%2) + psrlx mx, %2+%3+%4 +IF Y, psrlx my, BITS - %2 +IF Z, psrlx mz, BITS - %3 +IF W, psrlx mw, BITS - %4 + %if V2 +IF W, psllx mw2, mx2, BITS - (%4) +IF Z, psllx mz2, mx2, BITS - (%4+%3) +IF Y, psllx my2, mx2, BITS - (%4+%3+%2) + psrlx mx2, %2+%3+%4 +IF Y, psrlx my2, BITS - %2 +IF Z, psrlx mz2, BITS - %3 +IF W, psrlx mw2, BITS - %4 + %endif %endif CONTINUE tmp0q %endmacro -;--------------------------------------------------------- -; Clearing - -%macro clear_alpha 3 ; idx, vreg, vreg2 -op clear_alpha%1 - LOAD_CONT tmp0q - pcmpeqb %2, %2 -IF V2, mova %3, %2 - CONTINUE tmp0q -%endmacro - -%macro clear_zero 3 ; idx, vreg, vreg2 -op clear_zero%1 - LOAD_CONT tmp0q - pxor %2, %2 -IF V2, mova %3, %2 - CONTINUE tmp0q -%endmacro - -%macro clear_generic 0 -op clear - LOAD_CONT tmp0q -%if cpuflag(avx2) - IF X, vpbroadcastd mx, [implq + SwsOpImpl.priv + 0] - IF Y, vpbroadcastd my, [implq + SwsOpImpl.priv + 4] - IF Z, vpbroadcastd mz, [implq + SwsOpImpl.priv + 8] - IF W, vpbroadcastd mw, [implq + SwsOpImpl.priv + 12] -%else ; !cpuflag(avx2) - IF X, movd mx, [implq + SwsOpImpl.priv + 0] - IF Y, movd my, [implq + SwsOpImpl.priv + 4] - IF Z, movd mz, [implq + SwsOpImpl.priv + 8] - IF W, movd mw, [implq + SwsOpImpl.priv + 12] - IF X, pshufd mx, mx, 0 - IF Y, pshufd my, my, 0 - IF Z, pshufd mz, mz, 0 - IF W, pshufd mw, mw, 0 -%endif -%if V2 - IF X, mova mx2, mx - IF Y, mova my2, my - IF Z, mova mz2, mz - IF W, mova mw2, mw -%endif - CONTINUE tmp0q -%endmacro - -%macro clear_funcs 0 - decl_pattern 0, 0, 0, 1, clear_generic - decl_pattern 1, 0, 0, 0, clear_generic - decl_pattern 1, 1, 0, 0, clear_generic - decl_pattern 0, 1, 0, 0, clear_generic - decl_pattern 0, 1, 1, 0, clear_generic - decl_pattern 0, 0, 1, 1, clear_generic - decl_pattern 1, 0, 1, 0, clear_generic - decl_pattern 0, 1, 0, 1, clear_generic - decl_pattern 0, 1, 1, 1, clear_generic - decl_pattern 1, 0, 1, 1, clear_generic - decl_pattern 1, 1, 0, 1, clear_generic -%endmacro - -;--------------------------------------------------------- -; Swizzling and duplicating - -; mA := mB, mB := mC, ... mX := mA across both halves -%macro vrotate 2-* ; A, B, C, ... - %rep %0 - %assign rot_a %1 + 4 - %assign rot_b %2 + 4 - mova m%1, m%2 - IF V2, mova m%[rot_a], m%[rot_b] - %rotate 1 - %endrep - %undef rot_a - %undef rot_b -%endmacro - -%macro swizzle_funcs 0 -op swizzle_3012 - LOAD_CONT tmp0q - vrotate 8, 0, 3, 2, 1 - CONTINUE tmp0q - -op swizzle_3021 - LOAD_CONT tmp0q - vrotate 8, 0, 3, 1 - CONTINUE tmp0q - -op swizzle_2103 - LOAD_CONT tmp0q - vrotate 8, 0, 2 - CONTINUE tmp0q - -op swizzle_3210 - LOAD_CONT tmp0q - vrotate 8, 0, 3 - vrotate 8, 1, 2 - CONTINUE tmp0q - -op swizzle_3102 - LOAD_CONT tmp0q - vrotate 8, 0, 3, 2 - CONTINUE tmp0q - -op swizzle_3201 - LOAD_CONT tmp0q - vrotate 8, 0, 3, 1, 2 - CONTINUE tmp0q - -op swizzle_1203 - LOAD_CONT tmp0q - vrotate 8, 0, 1, 2 - CONTINUE tmp0q - -op swizzle_1023 - LOAD_CONT tmp0q - vrotate 8, 0, 1 - CONTINUE tmp0q - -op swizzle_2013 - LOAD_CONT tmp0q - vrotate 8, 0, 2, 1 - CONTINUE tmp0q - -op swizzle_2310 - LOAD_CONT tmp0q - vrotate 8, 0, 2, 1, 3 - CONTINUE tmp0q - -op swizzle_2130 - LOAD_CONT tmp0q - vrotate 8, 0, 2, 3 - CONTINUE tmp0q - -op swizzle_1230 - LOAD_CONT tmp0q - vrotate 8, 0, 1, 2, 3 - CONTINUE tmp0q - -op swizzle_1320 - LOAD_CONT tmp0q - vrotate 8, 0, 1, 3 - CONTINUE tmp0q - -op swizzle_0213 - LOAD_CONT tmp0q - vrotate 8, 1, 2 - CONTINUE tmp0q - -op swizzle_0231 - LOAD_CONT tmp0q - vrotate 8, 1, 2, 3 - CONTINUE tmp0q - -op swizzle_0312 - LOAD_CONT tmp0q - vrotate 8, 1, 3, 2 - CONTINUE tmp0q - -op swizzle_3120 - LOAD_CONT tmp0q - vrotate 8, 0, 3 - CONTINUE tmp0q - -op swizzle_0321 - LOAD_CONT tmp0q - vrotate 8, 1, 3 - CONTINUE tmp0q - -op swizzle_0003 - LOAD_CONT tmp0q - mova my, mx - mova mz, mx -%if V2 - mova my2, mx2 - mova mz2, mx2 -%endif - CONTINUE tmp0q - -op swizzle_0001 - LOAD_CONT tmp0q - mova mw, my - mova mz, mx - mova my, mx -%if V2 - mova mw2, my2 - mova mz2, mx2 - mova my2, mx2 -%endif - CONTINUE tmp0q - -op swizzle_3000 - LOAD_CONT tmp0q - mova my, mx - mova mz, mx - mova mx, mw - mova mw, my -%if V2 - mova my2, mx2 - mova mz2, mx2 - mova mx2, mw2 - mova mw2, my2 -%endif - CONTINUE tmp0q - -op swizzle_1000 - LOAD_CONT tmp0q - mova mz, mx - mova mw, mx - mova mx, my - mova my, mz -%if V2 - mova mz2, mx2 - mova mw2, mx2 - mova mx2, my2 - mova my2, mz2 -%endif - CONTINUE tmp0q -%endmacro - ;--------------------------------------------------------- ; Pixel type conversions -%macro conv8to16 1 ; type -op %1_U8_U16 - LOAD_CONT tmp0q -%if V2 - %if cpuflag(avx2) - IF X, vextracti128 xmx2, mx, 1 - IF Y, vextracti128 xmy2, my, 1 - IF Z, vextracti128 xmz2, mz, 1 - IF W, vextracti128 xmw2, mw, 1 - %else - IF X, psrldq xmx2, mx, 8 - IF Y, psrldq xmy2, my, 8 - IF Z, psrldq xmz2, mz, 8 - IF W, psrldq xmw2, mw, 8 - %endif - IF X, pmovzxbw mx2, xmx2 - IF Y, pmovzxbw my2, xmy2 - IF Z, pmovzxbw mz2, xmz2 - IF W, pmovzxbw mw2, xmw2 -%endif ; V2 - IF X, pmovzxbw mx, xmx - IF Y, pmovzxbw my, xmy - IF Z, pmovzxbw mz, xmz - IF W, pmovzxbw mw, xmw - -%ifidn %1, expand - VBROADCASTI128 m8, [expand16_shuf] - %if V2 - IF X, pshufb mx2, m8 - IF Y, pshufb my2, m8 - IF Z, pshufb mz2, m8 - IF W, pshufb mw2, m8 - %endif - IF X, pshufb mx, m8 - IF Y, pshufb my, m8 - IF Z, pshufb mz, m8 - IF W, pshufb mw, m8 -%endif ; expand - CONTINUE tmp0q +%macro cast8to16 4 ; reg, reg2, xreg, xreg2 +IF V2, vextracti128 %4, %1, 1 + pmovzxbw %1, %3 +IF V2, pmovzxbw %2, %4 %endmacro -%macro conv16to8 0 -op convert_U16_U8 - LOAD_CONT tmp0q +%macro cast16to8 4 ; reg, reg2, xreg, xreg2 %if V2 - ; this code technically works for the !V2 case as well, but slower -IF X, packuswb mx, mx2 -IF Y, packuswb my, my2 -IF Z, packuswb mz, mz2 -IF W, packuswb mw, mw2 -IF X, vpermq mx, mx, q3120 -IF Y, vpermq my, my, q3120 -IF Z, vpermq mz, mz, q3120 -IF W, vpermq mw, mw, q3120 + packuswb %1, %2 + vpermq %1, %1, q3120 %else -IF X, vextracti128 xm8, mx, 1 -IF Y, vextracti128 xm9, my, 1 -IF Z, vextracti128 xm10, mz, 1 -IF W, vextracti128 xm11, mw, 1 - vzeroupper -IF X, packuswb xmx, xm8 -IF Y, packuswb xmy, xm9 -IF Z, packuswb xmz, xm10 -IF W, packuswb xmw, xm11 + vextracti128 %4, %1, 1 + packuswb %3, %4 +%endif +%endmacro + +%macro cast8to32 4 ; reg, reg2, xreg, xreg2 + psrldq %4, %3, 8 + pmovzxbd %1, %3 + pmovzxbd %2, %4 +%endmacro + +%macro cast32to8 4 ; reg, reg2, xreg, xreg2 + packusdw %1, %2 + vextracti128 %4, %1, 1 + packuswb %3, %4 + vpshufd %3, %3, q3120 +%endmacro + +%macro cast16to32 4 ; reg, reg2, xreg, xreg2 + vextracti128 %4, %1, 1 + pmovzxwd %1, %3 + pmovzxwd %2, %4 +%endmacro + +%macro cast32to16 4 ; reg, reg2, xreg, xreg2 + packusdw %1, %2 + vpermq %1, %1, q3120 +%endmacro + +%macro CAST_TO 0 +%ifidn UOP, SWS_UOP_TO_U8 + %assign BITS_TO 8 +%elifidn UOP, SWS_UOP_TO_U16 + %assign BITS_TO 16 +%else + %assign BITS_TO 32 +%endif + + LOAD_CONT tmp0q +%ifidn TYPE, SWS_PIXEL_F32 +IF X, cvttps2dq mx, mx +IF Y, cvttps2dq my, my +IF Z, cvttps2dq mz, mz +IF W, cvttps2dq mw, mw +IF X, cvttps2dq mx2, mx2 +IF Y, cvttps2dq my2, my2 +IF Z, cvttps2dq mz2, mz2 +IF W, cvttps2dq mw2, mw2 +%endif +%if BITS != BITS_TO + ; integer size conversion +IF1 X, cast %+ BITS %+ to %+ BITS_TO mx, mx2, xmx, xmx2 +IF1 Y, cast %+ BITS %+ to %+ BITS_TO my, my2, xmy, xmy2 +IF1 Z, cast %+ BITS %+ to %+ BITS_TO mz, mz2, xmz, xmz2 +IF1 W, cast %+ BITS %+ to %+ BITS_TO mw, mw2, xmw, xmw2 + %if cpuflag(avx2) && (BITS > BITS_TO && !V2 || BITS >= BITS_TO * 4) + ; clear upper bits after reducing the register size + vzeroupper ; TMP + %endif +%endif +%ifidn UOP, SWS_UOP_TO_F32 +IF X, vcvtdq2ps mx, mx +IF Y, vcvtdq2ps my, my +IF Z, vcvtdq2ps mz, mz +IF W, vcvtdq2ps mw, mw +IF X, vcvtdq2ps mx2, mx2 +IF Y, vcvtdq2ps my2, my2 +IF Z, vcvtdq2ps mz2, mz2 +IF W, vcvtdq2ps mw2, mw2 %endif CONTINUE tmp0q %endmacro -%macro conv8to32 1 ; type -op %1_U8_U32 +;--------------------------------------------------------- +; Permuting, copying and clearing + +%macro MOV_IDX 2 ; dstidx, srcidx + %assign DST2 4 + %1 + %assign SRC2 4 + %2 + mova m %+ %1, m %+ %2 +IF V2, mova m %+ DST2, m %+ SRC2 +%endmacro + +; decompose a permutation into unique cycles and emit a minimal set of mova +; instructions for each cycle +%macro PERMUTE_CYCLES 4 ; x, y, z, w +%assign IN0 %1 +%assign IN1 %2 +%assign IN2 %3 +%assign IN3 %4 +%assign TMP 8 + LOAD_CONT tmp0q -IF X, psrldq xmx2, xmx, 8 -IF Y, psrldq xmy2, xmy, 8 -IF Z, psrldq xmz2, xmz, 8 -IF W, psrldq xmw2, xmw, 8 -IF X, pmovzxbd mx, xmx -IF Y, pmovzxbd my, xmy -IF Z, pmovzxbd mz, xmz -IF W, pmovzxbd mw, xmw -IF X, pmovzxbd mx2, xmx2 -IF Y, pmovzxbd my2, xmy2 -IF Z, pmovzxbd mz2, xmz2 -IF W, pmovzxbd mw2, xmw2 -%ifidn %1, expand - VBROADCASTI128 m8, [expand32_shuf] +%rep 4 + %ifndef CUR + ; start of new cycle, find next register not in correct location + %if X + %assign CUR IN0 + %assign IN0 TMP + %elif Y + %assign CUR IN1 + %assign IN1 TMP + %elif Z + %assign CUR IN2 + %assign IN2 TMP + %elif W + %assign CUR IN3 + %assign IN3 TMP + %else + %exitrep ; all registers happy + %endif + MOV_IDX TMP, CUR ; preserve previous value of CUR + %endif + %ifdef CUR ; work-around for NASM bug + ; rotate CUR <- in[CUR] and follow the cycle + %assign NEXT IN %+ CUR + MOV_IDX CUR, NEXT + + %if CUR == 0 + %assign X 0 + %elif CUR == 1 + %assign Y 0 + %elif CUR == 2 + %assign Z 0 + %else + %assign W 0 + %endif + + %assign CUR NEXT + %if CUR == TMP ; end of cycle + %assign TMP TMP+1 ; pick non-overlapping tmp register for next cycle + %undef CUR + %endif + %endif +%endrep + + CONTINUE tmp0q +%endmacro + +%macro COPY 4 ; x, y, z, w + LOAD_CONT tmp0q +IF X, mova m8, m%1 +IF Y, mova m9, m%2 +IF Z, mova m10, m%3 +IF W, mova m11, m%4 +IF X, mova mx, m8 +IF Y, mova my, m9 +IF Z, mova mz, m10 +IF W, mova mw, m11 +%if V2 + %assign x2 4 + %1 + %assign y2 4 + %2 + %assign z2 4 + %3 + %assign w2 4 + %4 +IF X, mova m12, m %+ x2 +IF Y, mova m13, m %+ y2 +IF Z, mova m14, m %+ z2 +IF W, mova m15, m %+ w2 +IF X, mova mx2, m12 +IF Y, mova my2, m13 +IF Z, mova mz2, m14 +IF W, mova mw2, m15 +%endif + CONTINUE tmp0q +%endmacro + +%macro clear 3 ; idx, reg, reg2 +%if SWS_COMP_TEST(ZERO_MASK, %1) + pxor %2, %2 +%elif SWS_COMP_TEST(ONE_MASK, %1) + pcmpeqb %2, %2 +%elif cpuflag(avx) + vpbroadcastd %2, [implq + SwsOpImpl.priv + 4 * %1] +%else + movd %2, [implq + SwsOpImpl.priv + 4 * %1] + pshufd %2, %2, 0 +%endif +IF V2, mova %3, %2 +%endmacro + +%macro CLEAR 2 +%assign ONE_MASK %1 +%assign ZERO_MASK %2 +IF1 X, clear 0, mx, mx2 +IF1 Y, clear 1, my, my2 +IF1 Z, clear 2, mz, mz2 +IF1 W, clear 3, mw, mw2 + CONTINUE +%endmacro + +;--------------------------------------------------------- +; Bit manipulation + +%macro SWAP_BYTES 0 + VBROADCASTI128 m8, [swap %+ BITS] IF X, pshufb mx, m8 IF Y, pshufb my, m8 IF Z, pshufb mz, m8 IF W, pshufb mw, m8 +%if V2 IF X, pshufb mx2, m8 IF Y, pshufb my2, m8 IF Z, pshufb mz2, m8 IF W, pshufb mw2, m8 -%endif ; expand - CONTINUE tmp0q +%endif + CONTINUE %endmacro -%macro conv32to8 0 -op convert_U32_U8 - LOAD_CONT tmp0q -IF X, packusdw mx, mx2 -IF Y, packusdw my, my2 -IF Z, packusdw mz, mz2 -IF W, packusdw mw, mw2 -IF X, vextracti128 xmx2, mx, 1 -IF Y, vextracti128 xmy2, my, 1 -IF Z, vextracti128 xmz2, mz, 1 -IF W, vextracti128 xmw2, mw, 1 - vzeroupper -IF X, packuswb xmx, xmx2 -IF Y, packuswb xmy, xmy2 -IF Z, packuswb xmz, xmz2 -IF W, packuswb xmw, xmw2 -IF X, vpshufd xmx, xmx, q3120 -IF Y, vpshufd xmy, xmy, q3120 -IF Z, vpshufd xmz, xmz, q3120 -IF W, vpshufd xmw, xmw, q3120 - CONTINUE tmp0q -%endmacro - -%macro conv16to32 0 -op convert_U16_U32 - LOAD_CONT tmp0q -IF X, vextracti128 xmx2, mx, 1 -IF Y, vextracti128 xmy2, my, 1 -IF Z, vextracti128 xmz2, mz, 1 -IF W, vextracti128 xmw2, mw, 1 -IF X, pmovzxwd mx, xmx -IF Y, pmovzxwd my, xmy -IF Z, pmovzxwd mz, xmz -IF W, pmovzxwd mw, xmw -IF X, pmovzxwd mx2, xmx2 -IF Y, pmovzxwd my2, xmy2 -IF Z, pmovzxwd mz2, xmz2 -IF W, pmovzxwd mw2, xmw2 - CONTINUE tmp0q -%endmacro - -%macro conv32to16 0 -op convert_U32_U16 - LOAD_CONT tmp0q -IF X, packusdw mx, mx2 -IF Y, packusdw my, my2 -IF Z, packusdw mz, mz2 -IF W, packusdw mw, mw2 -IF X, vpermq mx, mx, q3120 -IF Y, vpermq my, my, q3120 -IF Z, vpermq mz, mz, q3120 -IF W, vpermq mw, mw, q3120 - CONTINUE tmp0q -%endmacro - -;--------------------------------------------------------- -; Shifting and scaling - -%macro lshift16 0 -op lshift16 - vmovq xm8, [implq + SwsOpImpl.priv] - LOAD_CONT tmp0q -IF X, psllw mx, xm8 -IF Y, psllw my, xm8 -IF Z, psllw mz, xm8 -IF W, psllw mw, xm8 +%macro EXPAND_BIT 0 +%if BITS == 8 + VBROADCASTI128 m8, [const1b] +%elif BITS == 16 + VBROADCASTI128 m8, [const1w] +%else + assert 0, EXPAND_BIT is not implemented for 32-bit types +%endif +IF X, pcmpeqx mx, m8 +IF Y, pcmpeqx my, m8 +IF Z, pcmpeqx mz, m8 +IF W, pcmpeqx mw, m8 %if V2 -IF X, psllw mx2, xm8 -IF Y, psllw my2, xm8 -IF Z, psllw mz2, xm8 -IF W, psllw mw2, xm8 +IF X, pcmpeqx mx2, m8 +IF Y, pcmpeqx my2, m8 +IF Z, pcmpeqx mz2, m8 +IF W, pcmpeqx mw2, m8 +%endif + CONTINUE +%endmacro + +%macro EXPAND_BYTE 0 + assert BITS == 8 +%ifidn UOP, SWS_UOP_EXPAND_PAIR + %assign BITS_TO BITS * 2 +%elifidn UOP, SWS_UOP_EXPAND_QUAD + %assign BITS_TO BITS * 4 +%endif + VBROADCASTI128 m8, [expand %+ BITS_TO] + LOAD_CONT tmp0q +IF1 X, cast8to %+ BITS_TO mx, mx2, xmx, xmx2 +IF1 Y, cast8to %+ BITS_TO my, my2, xmy, xmy2 +IF1 Z, cast8to %+ BITS_TO mz, mz2, xmz, xmz2 +IF1 W, cast8to %+ BITS_TO mw, mw2, xmw, xmw2 +IF X, pshufb mx, m8 +IF Y, pshufb my, m8 +IF Z, pshufb mz, m8 +IF W, pshufb mw, m8 + %if V2 +IF X, pshufb mx2, m8 +IF Y, pshufb my2, m8 +IF Z, pshufb mz2, m8 +IF W, pshufb mw2, m8 + %endif + CONTINUE tmp0q +%endmacro + +%macro LSHIFT 1 ; shift + LOAD_CONT tmp0q +IF X, psllx mx, %1 +IF Y, psllx my, %1 +IF Z, psllx mz, %1 +IF W, psllx mw, %1 +%if V2 +IF X, psllx mx2, %1 +IF Y, psllx my2, %1 +IF Z, psllx mz2, %1 +IF W, psllx mw2, %1 %endif CONTINUE tmp0q %endmacro -%macro rshift16 0 -op rshift16 - vmovq xm8, [implq + SwsOpImpl.priv] +%macro RSHIFT 1 ; shift LOAD_CONT tmp0q -IF X, psrlw mx, xm8 -IF Y, psrlw my, xm8 -IF Z, psrlw mz, xm8 -IF W, psrlw mw, xm8 +IF X, psrlx mx, %1 +IF Y, psrlx my, %1 +IF Z, psrlx mz, %1 +IF W, psrlx mw, %1 %if V2 -IF X, psrlw mx2, xm8 -IF Y, psrlw my2, xm8 -IF Z, psrlw mz2, xm8 -IF W, psrlw mw2, xm8 +IF X, psrlx mx2, %1 +IF Y, psrlx my2, %1 +IF Z, psrlx mz2, %1 +IF W, psrlx mw2, %1 %endif CONTINUE tmp0q %endmacro -; special cases for expanding bits to full range -%macro expand_bits 2 ; bits, suffix -op expand_bits%1 - mova m8, [const1%2] +;--------------------------------------------------------- +; Arithmetic operations + +%macro pmull4 6 ; suffix, x, y, z, w, k +IF X, pmull%1 %2, %6 +IF Y, pmull%1 %3, %6 +IF Z, pmull%1 %4, %6 +IF W, pmull%1 %5, %6 +%endmacro + +%macro scale8 5 ; mx, my, mz, mw, k +IF X, mova m8, %1 +IF Y, mova m9, %2 +IF Z, mova m10, %3 +IF W, mova m11, %4 +IF X, punpcklbw %1, m15 +IF Y, punpcklbw %2, m15 +IF Z, punpcklbw %3, m15 +IF W, punpcklbw %4, m15 +IF X, punpckhbw m8, m15 +IF Y, punpckhbw m9, m15 +IF Z, punpckhbw m10, m15 +IF W, punpckhbw m11, m15 + pmull4 w, %1, %2, %3, %4, m12 + pmull4 w, m8, m9, m10, m11, m12 +IF X, packuswb %1, m8 +IF Y, packuswb %2, m9 +IF Z, packuswb %3, m10 +IF W, packuswb %4, m11 +%endmacro + +%macro SCALE 0 LOAD_CONT tmp0q - pcmpeq%2 mx, m8 -IF V2, pcmpeq%2 mx2, m8 +%if BITS == 8 + vpbroadcastw m12, [implq + SwsOpImpl.priv] + pxor m15, m15 + scale8 mx, my, mz, mw, m12 +IF1 V2, scale8 mx2, my2, mz2, mw2, m12 +%else + vpbroadcastx m12, [implq + SwsOpImpl.priv] + pmull4 x, mx, my, mz, mw, m12 +IF1 V2, pmull4 x, mx2, my2, mz2, mw2, m12 +%endif CONTINUE tmp0q %endmacro -;--------------------------------------------------------- -; Macro instantiations for kernel functions - -%macro funcs_u8 0 - read_planar 1 - read_planar 2 - read_planar 3 - read_planar 4 - write_planar 1 - write_planar 2 - write_planar 3 - write_planar 4 - - rw_packed 8 - read_nibbles - read_bits - write_bits - expand_bits 8, b - - pack_generic 1, 2, 1 - pack_generic 3, 3, 2 - pack_generic 2, 3, 3 - unpack8 1, 2, 1 - unpack8 3, 3, 2 - unpack8 2, 3, 3 - - clear_alpha 0, mx, mx2 - clear_alpha 1, my, my2 - clear_alpha 3, mw, mw2 - clear_zero 0, mx, mx2 - clear_zero 1, my, my2 - clear_zero 3, mw, mw2 - clear_funcs - swizzle_funcs - - decl_common_patterns shuffle +%macro ADD 0 +assert 0, SWS_UOP_ADD is not implemented for integer types %endmacro -%macro funcs_u16 0 - rw_packed 16 - expand_bits 16, w - pack_generic 4, 4, 4 - pack_generic 5, 5, 5 - pack_generic 5, 6, 5 - unpack w, 16, 4, 4, 4 - unpack w, 16, 5, 5, 5 - unpack w, 16, 5, 6, 5 - decl_common_patterns conv8to16 convert - decl_common_patterns conv8to16 expand - decl_common_patterns conv16to8 - decl_common_patterns lshift16 - decl_common_patterns rshift16 +%macro MIN 0 +assert 0, SWS_UOP_MIN is not implemented for integer types +%endmacro + +%macro MAX 0 +assert 0, SWS_UOP_MAX is not implemented for integer types +%endmacro + +%macro LINEAR_FMA 0-* +assert 0, SWS_UOP_LINEAR_FMA is not implemented for integer types +%endmacro + +%macro DITHER 0-* +assert 0, SWS_UOP_DITHER is not implemented for integer types +%endmacro + +;--------------------------------------------------------- +; Instantiate above macros to generate all uop kernels + +%macro decl_ops 1 ; type + DECL_%1_READ_PACKED (READ_PACKED) + DECL_%1_READ_NIBBLE (READ_NIBBLE) + DECL_%1_READ_BIT (READ_BIT) + DECL_%1_WRITE_PACKED (WRITE_PACKED) + DECL_%1_WRITE_NIBBLE (WRITE_NIBBLE) + DECL_%1_WRITE_BIT (WRITE_BIT) + DECL_%1_PERMUTE (PERMUTE_CYCLES) + DECL_%1_COPY (COPY) + DECL_%1_SWAP_BYTES (SWAP_BYTES) + DECL_%1_EXPAND_BIT (EXPAND_BIT) + DECL_%1_SCALE (SCALE) + DECL_%1_ADD (ADD) + DECL_%1_MIN (MIN) + DECL_%1_MAX (MAX) + DECL_%1_UNPACK (UNPACK) + DECL_%1_PACK (PACK) + DECL_%1_LSHIFT (LSHIFT) + DECL_%1_RSHIFT (RSHIFT) + DECL_%1_LINEAR_FMA (LINEAR_FMA) + DECL_%1_DITHER (DITHER) +%endmacro + +%macro decl_type_invariant 0 + DECL_U8_READ_PLANAR (READ_PLANAR) + DECL_U8_WRITE_PLANAR (WRITE_PLANAR) + DECL_U8_CLEAR (CLEAR) +%endmacro + +%macro decl_cast_u16 0 + DECL_U8_TO_U16 (CAST_TO) + DECL_U16_TO_U8 (CAST_TO) + DECL_U8_EXPAND_PAIR (EXPAND_BYTE) +%endmacro + +%macro decl_cast_u32 0 + DECL_U8_TO_U32 (CAST_TO) + DECL_U32_TO_U8 (CAST_TO) + DECL_U16_TO_U32 (CAST_TO) + DECL_U32_TO_U16 (CAST_TO) + DECL_U8_EXPAND_QUAD (EXPAND_BYTE) +%endmacro + +%macro decl_cast_f32 0 + DECL_U8_TO_F32 (CAST_TO) + DECL_F32_TO_U8 (CAST_TO) + DECL_U16_TO_F32 (CAST_TO) + DECL_F32_TO_U16 (CAST_TO) + DECL_U32_TO_F32 (CAST_TO) + DECL_F32_TO_U32 (CAST_TO) %endmacro INIT_XMM sse4 -decl_v2 0, funcs_u8 -decl_v2 1, funcs_u8 +decl_v2 0, decl_ops U8 +decl_v2 1, decl_ops U8 +decl_v2 0, decl_type_invariant +decl_v2 1, decl_type_invariant INIT_YMM avx2 -decl_v2 0, funcs_u8 -decl_v2 1, funcs_u8 -decl_v2 0, funcs_u16 -decl_v2 1, funcs_u16 +decl_v2 0, decl_ops U8 +decl_v2 1, decl_ops U8 +decl_v2 0, decl_ops U16 +decl_v2 1, decl_ops U16 +decl_v2 0, decl_type_invariant +decl_v2 1, decl_type_invariant +decl_v2 0, decl_cast_u16 +decl_v2 1, decl_cast_u16 -INIT_YMM avx2 -decl_v2 1, rw_packed 32 -decl_v2 1, pack_generic 10, 10, 10, 2 -decl_v2 1, pack_generic 2, 10, 10, 10 -decl_v2 1, unpack d, 32, 10, 10, 10, 2 -decl_v2 1, unpack d, 32, 2, 10, 10, 10 -decl_common_patterns conv8to32 convert -decl_common_patterns conv8to32 expand -decl_common_patterns conv32to8 -decl_common_patterns conv16to32 -decl_common_patterns conv32to16 +decl_v2 1, decl_ops U32 +decl_v2 1, decl_cast_u32 +decl_v2 1, decl_cast_f32