From 508b82c605b0e53ffcb3c92a3ceac225cbf32f79 Mon Sep 17 00:00:00 2001 From: Devin Heitmueller Date: Wed, 16 Sep 2026 15:48:38 -0400 Subject: [PATCH] avcodec/bitpacked: add x86 SIMD unpacking Add runtime-dispatched SSSE3 and AVX2 implementations for packed 10-bit YUV 4:2:2. The SSSE3 loop follows the negative-width output-offset pattern used by v210. The AVX2 loop processes 16 pixels at a time and has an internal eight-pixel tail. Both implementations use exact-width input loads and therefore do not rely on padded input or legal overread. They passed 300 consecutive checkasm seeds. Benchmark command: ./tests/checkasm/checkasm --bench --test=bitpackeddec \ --duration=1000000 51000 Intel Xeon E-2356G, GCC 8.5, NASM 2.15.03: bitpacked_unpack_yuv422p10_c: 13516.7 bitpacked_unpack_yuv422p10_ssse3: 3415.8 (3.96x) bitpacked_unpack_yuv422p10_avx2: 2183.1 (6.19x) --- libavcodec/bitpacked_dec.h | 1 + libavcodec/bitpacked_dec_init.h | 2 + libavcodec/x86/Makefile | 2 + libavcodec/x86/bitpacked.asm | 186 ++++++++++++++++++++++++++++++++ libavcodec/x86/bitpacked_init.c | 47 ++++++++ 5 files changed, 238 insertions(+) create mode 100644 libavcodec/x86/bitpacked.asm create mode 100644 libavcodec/x86/bitpacked_init.c diff --git a/libavcodec/bitpacked_dec.h b/libavcodec/bitpacked_dec.h index 637f3789ef..cd70ab1318 100644 --- a/libavcodec/bitpacked_dec.h +++ b/libavcodec/bitpacked_dec.h @@ -37,5 +37,6 @@ struct BitpackedContext { }; void ff_bitpacked_init_aarch64(struct BitpackedContext *s); +void ff_bitpacked_init_x86(struct BitpackedContext *s); #endif /* AVCODEC_BITPACKED_DEC_H */ diff --git a/libavcodec/bitpacked_dec_init.h b/libavcodec/bitpacked_dec_init.h index f0b19f04fa..ee0b4d8f96 100644 --- a/libavcodec/bitpacked_dec_init.h +++ b/libavcodec/bitpacked_dec_init.h @@ -48,6 +48,8 @@ av_unused static av_cold void ff_bitpackeddec_init(struct BitpackedContext *s) #if ARCH_AARCH64 ff_bitpacked_init_aarch64(s); +#elif ARCH_X86 && HAVE_X86ASM + ff_bitpacked_init_x86(s); #endif } diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile index 456a209482..b1b38ef8aa 100644 --- a/libavcodec/x86/Makefile +++ b/libavcodec/x86/Makefile @@ -48,6 +48,7 @@ X86ASM-OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp_init.o X86ASM-OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp_init.o X86ASM-OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp_init.o X86ASM-OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp_init.o +X86ASM-OBJS-$(CONFIG_BITPACKED_DECODER) += x86/bitpacked_init.o X86ASM-OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsdsp.o X86ASM-OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp_init.o X86ASM-OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp_init.o @@ -150,6 +151,7 @@ X86ASM-OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp.o X86ASM-OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp.o X86ASM-OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp.o X86ASM-OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp.o +X86ASM-OBJS-$(CONFIG_BITPACKED_DECODER) += x86/bitpacked.o X86ASM-OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsidct.o x86/cavs_qpel.o X86ASM-OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp.o X86ASM-OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp.o diff --git a/libavcodec/x86/bitpacked.asm b/libavcodec/x86/bitpacked.asm new file mode 100644 index 0000000000..db15a2f91f --- /dev/null +++ b/libavcodec/x86/bitpacked.asm @@ -0,0 +1,186 @@ +;****************************************************************************** +;* Bitpacked 10-bit YUV 4:2:2 unpacking +;* Copyright (c) 2026 Devin Heitmueller +;* +;* This file is part of FFmpeg. +;* +;* FFmpeg is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* FFmpeg is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with FFmpeg; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +SECTION_RODATA 32 + +unpack_shuf_avx2: + db 1,0, 2,1, 3,2, 4,3, 6,5, 7,6, 8,7, 9,8 + db 5,4, 6,5, 7,6, 8,7, 10,9, 11,10, 12,11, 13,12 +u_shuf_ymm: + db 0,1, 8,9, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + db 0,1, 8,9, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +v_shuf_ymm: + db 4,5, 12,13, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + db 4,5, 12,13, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + +unpack_shuf: db 1,0, 2,1, 3,2, 4,3, 6,5, 7,6, 8,7, 9,8 +unpack_mult: dw 1,4,16,64, 1,4,16,64 +y_shuf: db 2,3, 6,7, 10,11, 14,15, -1,-1,-1,-1,-1,-1,-1,-1 +u_shuf: db 0,1, 8,9, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +v_shuf: db 4,5, 12,13, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 + +SECTION .text + +; ff_bitpacked_unpack_yuv422p10(const uint8_t *src, +; uint16_t *y, uint16_t *u, uint16_t *v, +; int width) +; Form four big-endian 16-bit windows per five input bytes. Multiplication by +; 1, 4, 16 and 64 aligns all four samples for a common six-bit right shift. +INIT_XMM ssse3 +cglobal bitpacked_unpack_yuv422p10, 5, 5, 8, src, y, u, v, w + movsxdifnidn wq, wd + lea yq, [yq+2*wq] + add uq, wq + add vq, wq + neg wq + + mova m4, [unpack_shuf] + mova m5, [unpack_mult] + mova m6, [y_shuf] + mova m7, [u_shuf] + +.loop: + movq m0, [srcq] + pinsrw m0, [srcq+8], 4 + movq m1, [srcq+10] + pinsrw m1, [srcq+18], 4 + + pshufb m0, m4 + pshufb m1, m4 + pmullw m0, m5 + pmullw m1, m5 + psrlw m0, 6 + psrlw m1, 6 + + mova m2, m0 + mova m3, m1 + pshufb m2, m6 + pshufb m3, m6 + punpcklqdq m2, m3 + movu [yq+2*wq], m2 + + mova m2, m0 + mova m3, m1 + pshufb m2, m7 + pshufb m3, m7 + punpckldq m2, m3 + movq [uq+wq], m2 + + pshufb m0, [v_shuf] + pshufb m1, [v_shuf] + punpckldq m0, m1 + movq [vq+wq], m0 + + add srcq, 20 + add wq, 8 + jl .loop + + RET + +%if HAVE_AVX2_EXTERNAL +%macro unpack_16 0 + ; m0 contains the first and third 10-byte groups; m1 the second and fourth. + movu m0, [srcq] + movq xm1, [srcq+10] + pinsrw xm1, [srcq+18], 4 + movq xm2, [srcq+30] + pinsrw xm2, [srcq+38], 4 + vinserti128 m1, m1, xm2, 1 + + pshufb m0, m4 + pshufb m1, m5 + pmullw m0, m6 + pmullw m1, m6 + psrlw m0, 6 + psrlw m1, 6 + + pshufb m2, m0, m7 + pshufb m3, m1, m7 + punpcklqdq m2, m3 + movu [yq], m2 + + pshufb m2, m0, [u_shuf_ymm] + pshufb m3, m1, [u_shuf_ymm] + punpckldq m2, m3 + vpermq m2, m2, 0xd8 + movu [uq], xm2 + + pshufb m2, m0, [v_shuf_ymm] + pshufb m3, m1, [v_shuf_ymm] + punpckldq m2, m3 + vpermq m2, m2, 0xd8 + movu [vq], xm2 + + add srcq, 40 + add yq, 32 + add uq, 16 + add vq, 16 +%endmacro + +INIT_YMM avx2 +cglobal bitpacked_unpack_yuv422p10, 5, 5, 8, src, y, u, v, w + mova m4, [unpack_shuf_avx2] + VBROADCASTI128 m5, [unpack_shuf] + VBROADCASTI128 m6, [unpack_mult] + VBROADCASTI128 m7, [y_shuf] + + sub wd, 16 + jl .tail +.loop: + unpack_16 + sub wd, 16 + jge .loop + +.tail: + add wd, 16 + jz .ret + movq xm0, [srcq] + pinsrw xm0, [srcq+8], 4 + movq xm1, [srcq+10] + pinsrw xm1, [srcq+18], 4 + vinserti128 m0, m0, xm1, 1 + + pshufb m0, m5 + pmullw m0, m6 + psrlw m0, 6 + + pshufb m1, m0, m7 + vextracti128 xm2, m1, 1 + punpcklqdq xm1, xm2 + movu [yq], xm1 + + VBROADCASTI128 m3, [u_shuf] + pshufb m1, m0, m3 + vextracti128 xm2, m1, 1 + punpckldq xm1, xm2 + movq [uq], xm1 + + VBROADCASTI128 m3, [v_shuf] + pshufb m1, m0, m3 + vextracti128 xm2, m1, 1 + punpckldq xm1, xm2 + movq [vq], xm1 + +.ret: + RET +%endif diff --git a/libavcodec/x86/bitpacked_init.c b/libavcodec/x86/bitpacked_init.c new file mode 100644 index 0000000000..01fce4ba16 --- /dev/null +++ b/libavcodec/x86/bitpacked_init.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026 Devin Heitmueller + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include "libavutil/attributes.h" +#include "libavutil/x86/cpu.h" +#include "libavcodec/bitpacked_dec.h" + +void ff_bitpacked_unpack_yuv422p10_ssse3(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width); +#if HAVE_AVX2_EXTERNAL +void ff_bitpacked_unpack_yuv422p10_avx2(const uint8_t *src, + uint16_t *y, uint16_t *u, uint16_t *v, + int width); +#endif + +av_cold void ff_bitpacked_init_x86(struct BitpackedContext *s) +{ + int cpu_flags = av_get_cpu_flags(); + + if (EXTERNAL_SSSE3(cpu_flags)) + s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_ssse3; + +#if HAVE_AVX2_EXTERNAL + if (EXTERNAL_AVX2(cpu_flags)) + s->unpack_yuv422p10 = ff_bitpacked_unpack_yuv422p10_avx2; +#endif +}