;****************************************************************************** ;* Copyright Nick Kurshev ;* Copyright Michael (michaelni@gmx.at) ;* Copyright 2018 Jokyo Images ;* Copyright Ivo van Poorten ;* ;* This file is part of FFmpeg. ;* ;* FFmpeg is free software; you can redistribute it and/or ;* modify it under the terms of the GNU Lesser General Public ;* License as published by the Free Software Foundation; either ;* version 2.1 of the License, or (at your option) any later version. ;* ;* FFmpeg is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ;* Lesser General Public License for more details. ;* ;* You should have received a copy of the GNU Lesser General Public ;* License along with FFmpeg; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** %include "libavutil/x86/x86util.asm" SECTION_RODATA pb_shuffle2103: db 2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15 pb_shuffle0321: db 0, 3, 2, 1, 4, 7, 6, 5, 8, 11, 10, 9, 12, 15, 14, 13 pb_shuffle1230: db 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12 pb_shuffle3012: db 3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14 pb_shuffle3210: db 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 pb_shuffle3102: db 3, 1, 0, 2, 7, 5, 4, 6, 11, 9, 8, 10, 15, 13, 12, 14 pb_shuffle2013: db 2, 0, 1, 3, 6, 4, 5, 7, 10, 8, 9, 11, 14, 12, 13, 15 pb_shuffle2130: db 2, 1, 3, 0, 6, 5, 7, 4, 10, 9, 11, 8, 14, 13, 15, 12 pb_shuffle1203: db 1, 2, 0, 3, 5, 6, 4, 7, 9, 10, 8, 11, 13, 14, 12, 15 %if HAVE_AVX512ICL_EXTERNAL ; shuffle vector to rearrange packuswb result to be linear shuf_packus: db 0, 1, 2, 3, 16, 17, 18, 19, 32, 33, 34, 35, 48, 49, 50, 51,\ 4, 5, 6, 7, 20, 21, 22, 23, 36, 37, 38, 39, 52, 53, 54, 55,\ 8, 9, 10, 11, 24, 25, 26, 27, 40, 41, 42, 43, 56, 57, 58, 59,\ 12, 13, 14, 15, 28, 29, 30, 31, 44, 45, 46, 47, 60, 61, 62, 63 ; shuffle vector to combine odd elements from two vectors to extract Y shuf_perm2b: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,\ 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,\ 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95,\ 97, 99, 101, 103, 105, 107, 109, 111, 113, 115, 117, 119, 121, 123, 125, 127 %endif %if HAVE_AVX2_EXTERNAL ; shuffle vector to rearrange packuswb result to be linear shuf_packus_avx2: db 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0,\ 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, %endif SECTION .text %macro RSHIFT_COPY 3 ; %1 dst ; %2 src ; %3 shift %if cpuflag(avx) || cpuflag(avx2) || cpuflag(avx512icl) psrldq %1, %2, %3 %else mova %1, %2 RSHIFT %1, %3 %endif %endmacro ;------------------------------------------------------------------------------ ; shuffle_bytes_## (const uint8_t *src, uint8_t *dst, int src_size) ;------------------------------------------------------------------------------ ; %1-4 index shuffle %macro SHUFFLE_BYTES 4 cglobal shuffle_bytes_%1%2%3%4, 3, 5, 2, src, dst, w, tmp, x VBROADCASTI128 m0, [pb_shuffle%1%2%3%4] movsxdifnidn wq, wd mov xq, wq add srcq, wq add dstq, wq neg wq %if mmsize == 64 and xq, mmsize - 4 shr xq, 2 mov tmpd, -1 shlx tmpd, tmpd, xd not tmpd kmovw k7, tmpd vmovdqu32 m1{k7}{z}, [srcq + wq] pshufb m1, m0 vmovdqu32 [dstq + wq]{k7}, m1 lea wq, [wq + 4 * xq] %else ;calc scalar loop and xq, mmsize-4 je .loop_simd .loop_scalar: mov tmpb, [srcq + wq + %1] mov [dstq+wq + 0], tmpb mov tmpb, [srcq + wq + %2] mov [dstq+wq + 1], tmpb mov tmpb, [srcq + wq + %3] mov [dstq+wq + 2], tmpb mov tmpb, [srcq + wq + %4] mov [dstq+wq + 3], tmpb add wq, 4 sub xq, 4 jg .loop_scalar %endif ;check if src_size < mmsize cmp wq, 0 jge .end .loop_simd: movu m1, [srcq + wq] pshufb m1, m0 movu [dstq + wq], m1 add wq, mmsize jl .loop_simd .end: RET %endmacro INIT_XMM ssse3 SHUFFLE_BYTES 2, 1, 0, 3 SHUFFLE_BYTES 0, 3, 2, 1 SHUFFLE_BYTES 1, 2, 3, 0 SHUFFLE_BYTES 3, 0, 1, 2 SHUFFLE_BYTES 3, 2, 1, 0 SHUFFLE_BYTES 3, 1, 0, 2 SHUFFLE_BYTES 2, 0, 1, 3 SHUFFLE_BYTES 2, 1, 3, 0 SHUFFLE_BYTES 1, 2, 0, 3 %if ARCH_X86_64 %if HAVE_AVX2_EXTERNAL INIT_YMM avx2 SHUFFLE_BYTES 2, 1, 0, 3 SHUFFLE_BYTES 0, 3, 2, 1 SHUFFLE_BYTES 1, 2, 3, 0 SHUFFLE_BYTES 3, 0, 1, 2 SHUFFLE_BYTES 3, 2, 1, 0 SHUFFLE_BYTES 3, 1, 0, 2 SHUFFLE_BYTES 2, 0, 1, 3 SHUFFLE_BYTES 2, 1, 3, 0 SHUFFLE_BYTES 1, 2, 0, 3 %endif %endif %if ARCH_X86_64 %if HAVE_AVX512ICL_EXTERNAL INIT_ZMM avx512icl SHUFFLE_BYTES 2, 1, 0, 3 SHUFFLE_BYTES 0, 3, 2, 1 SHUFFLE_BYTES 1, 2, 3, 0 SHUFFLE_BYTES 3, 0, 1, 2 SHUFFLE_BYTES 3, 2, 1, 0 SHUFFLE_BYTES 3, 1, 0, 2 SHUFFLE_BYTES 2, 0, 1, 3 SHUFFLE_BYTES 2, 1, 3, 0 SHUFFLE_BYTES 1, 2, 0, 3 %endif %endif ;----------------------------------------------------------------------------------------------- ; uyvytoyuv422(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, ; const uint8_t *src, int width, int height, ; int lumStride, int chromStride, int srcStride) ;----------------------------------------------------------------------------------------------- %macro UYVY_TO_YUV422 0 cglobal uyvytoyuv422, 9, 14, 8 + cpuflag(avx2) + cpuflag(avx512icl), ydst, udst, vdst, src, w, h, lum_stride, chrom_stride, src_stride, wtwo, whalf, tmp, x, back_w pxor m0, m0 %if mmsize == 64 vpternlogd m1, m1, m1, 0xff ; m1 = _mm512_set1_epi8(0xff) movu m8, [shuf_packus] movu m9, [shuf_perm2b] %else %if cpuflag(avx2) movu m8, [shuf_packus_avx2] %endif pcmpeqw m1, m1 %endif psrlw m1, 8 movsxdifnidn wq, wd movsxdifnidn lum_strideq, lum_strided movsxdifnidn chrom_strideq, chrom_strided movsxdifnidn src_strideq, src_strided mov back_wq, wq mov whalfq, wq shr whalfq, 1 ; whalf = width / 2 lea srcq, [srcq + wq * 2] add ydstq, wq add udstq, whalfq add vdstq, whalfq .loop_line: mov xq, wq mov wtwoq, wq add wtwoq, wtwoq ; wtwo = width * 2 neg wq neg wtwoq neg whalfq ;calc scalar loop count and xq, mmsize * 2 - 1 je .loop_simd %if mmsize == 64 shr xq, 1 mov tmpq, -1 shlx tmpq, tmpq, xq not tmpq kmovq k7, tmpq ; write mask for U/V kmovd k1, tmpd ; write mask for 1st half of Y kmovw k3, tmpd ; read mask for 1st vector shr tmpq, 16 kmovw k4, tmpd ; read mask for 2nd vector shr tmpq, 16 kmovd k2, tmpd ; write mask for 2nd half of Y kmovw k5, tmpd ; read mask for 3rd vector shr tmpd, 16 kmovw k6, tmpd ; read mask for 4th vector vmovdqu32 m2{k3}{z}, [srcq + wtwoq ] vmovdqu32 m3{k4}{z}, [srcq + wtwoq + mmsize ] vmovdqu32 m4{k5}{z}, [srcq + wtwoq + mmsize * 2] vmovdqu32 m5{k6}{z}, [srcq + wtwoq + mmsize * 3] ; extract y part 1 mova m6, m9 vpermi2b m6, m2, m3 ; UYVY UYVY -> YYYY using permute vmovdqu16 [ydstq + wq]{k1}, m6 ; extract y part 2 mova m7, m9 vpermi2b m7, m4, m5 ; UYVY UYVY -> YYYY using permute vmovdqu16 [ydstq + wq + mmsize]{k2}, m7 ; extract uv pand m2, m1 ; UxVx... pand m3, m1 ; UxVx... pand m4, m1 ; UxVx... pand m5, m1 ; UxVx... packuswb m2, m3 ; UVUV... packuswb m4, m5 ; UVUV... ; U pand m6, m2, m1 ; UxUx... pand m7, m4, m1 ; UxUx... packuswb m6, m7 ; UUUU vpermb m6, m8, m6 vmovdqu8 [udstq + whalfq]{k7}, m6 ; V psrlw m2, 8 ; VxVx... psrlw m4, 8 ; VxVx... packuswb m2, m4 ; VVVV vpermb m2, m8, m2 vmovdqu8 [vdstq + whalfq]{k7}, m2 lea wq, [ wq + 2 * xq] lea wtwoq, [wtwoq + 4 * xq] add whalfq, xq %else .loop_scalar: mov tmpb, [srcq + wtwoq + 0] mov [udstq + whalfq], tmpb mov tmpb, [srcq + wtwoq + 1] mov [ydstq + wq], tmpb mov tmpb, [srcq + wtwoq + 2] mov [vdstq + whalfq], tmpb mov tmpb, [srcq + wtwoq + 3] mov [ydstq + wq + 1], tmpb add wq, 2 add wtwoq, 4 add whalfq, 1 sub xq, 2 jg .loop_scalar %endif ; check if simd loop is need cmp wq, 0 jge .end_line .loop_simd: movu m2, [srcq + wtwoq ] movu m3, [srcq + wtwoq + mmsize ] movu m4, [srcq + wtwoq + mmsize * 2] movu m5, [srcq + wtwoq + mmsize * 3] %if mmsize == 64 ; extract y part 1 mova m6, m9 vpermi2b m6, m2, m3 ; UYVY UYVY -> YYYY using permute movu [ydstq + wq], m6 ; extract y part 2 mova m7, m9 vpermi2b m7, m4, m5 ; UYVY UYVY -> YYYY using permute movu [ydstq + wq + mmsize], m7 %else ; extract y part 1 RSHIFT_COPY m6, m2, 1 ; UYVY UYVY -> YVYU YVY... pand m6, m1 ; YxYx YxYx... RSHIFT_COPY m7, m3, 1 ; UYVY UYVY -> YVYU YVY... pand m7, m1 ; YxYx YxYx... packuswb m6, m7 ; YYYY YYYY... %if mmsize == 32 vpermq m6, m6, 0xd8 %endif movu [ydstq + wq], m6 ; extract y part 2 RSHIFT_COPY m6, m4, 1 ; UYVY UYVY -> YVYU YVY... pand m6, m1 ; YxYx YxYx... RSHIFT_COPY m7, m5, 1 ; UYVY UYVY -> YVYU YVY... pand m7, m1 ; YxYx YxYx... packuswb m6, m7 ; YYYY YYYY... %if mmsize == 32 vpermq m6, m6, 0xd8 %endif movu [ydstq + wq + mmsize], m6 %endif ; extract uv pand m2, m1 ; UxVx... pand m3, m1 ; UxVx... pand m4, m1 ; UxVx... pand m5, m1 ; UxVx... packuswb m2, m3 ; UVUV... packuswb m4, m5 ; UVUV... ; U pand m6, m2, m1 ; UxUx... pand m7, m4, m1 ; UxUx... packuswb m6, m7 ; UUUU %if mmsize == 64 vpermb m6, m8, m6 %elif mmsize == 32 vpermd m6, m8, m6 %endif movu [udstq + whalfq], m6 ; V psrlw m2, 8 ; VxVx... psrlw m4, 8 ; VxVx... packuswb m2, m4 ; VVVV %if mmsize == 64 vpermb m2, m8, m2 %elif mmsize == 32 vpermd m2, m8, m2 %endif movu [vdstq + whalfq], m2 add whalfq, mmsize add wtwoq, mmsize * 4 add wq, mmsize * 2 jl .loop_simd .end_line: add srcq, src_strideq add ydstq, lum_strideq add udstq, chrom_strideq add vdstq, chrom_strideq ;restore initial state of line variable mov wq, back_wq mov xq, wq mov whalfq, wq shr whalfq, 1 ; whalf = width / 2 sub hd, 1 jg .loop_line RET %endmacro %if ARCH_X86_64 INIT_XMM sse2 UYVY_TO_YUV422 INIT_XMM avx UYVY_TO_YUV422 %if HAVE_AVX2_EXTERNAL INIT_YMM avx2 UYVY_TO_YUV422 %endif %if HAVE_AVX512ICL_EXTERNAL INIT_ZMM avx512icl UYVY_TO_YUV422 %endif %endif