/* * Copyright (c) 2024 Zhao Zhili * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "libavutil/aarch64/asm.S" #define VVC_MAX_PB_SIZE 128 #define BDOF_BLOCK_SIZE 16 #define BDOF_MIN_BLOCK_SIZE 4 .macro vvc_w_avg bit_depth .macro vvc_w_avg_\bit_depth\()_2_4 tap .if \tap == 2 ldr s0, [src0] ldr s2, [src1] .else ldr d0, [src0] ldr d2, [src1] .endif mov v4.16b, v16.16b smlal v4.4s, v0.4h, v19.4h smlal v4.4s, v2.4h, v20.4h sqshl v4.4s, v4.4s, v22.4s sqxtun v4.4h, v4.4s .if \bit_depth == 8 sqxtun v4.8b, v4.8h .if \tap == 2 str h4, [dst] .else // tap == 4 str s4, [dst] .endif .else // bit_depth > 8 umin v4.4h, v4.4h, v17.4h .if \tap == 2 str s4, [dst] .else str d4, [dst] .endif .endif add src0, src0, x10 add src1, src1, x10 add dst, dst, dst_stride .endm function ff_vvc_w_avg_\bit_depth\()_neon, export=1 dst .req x0 dst_stride .req x1 src0 .req x2 src1 .req x3 width .req w4 height .req w5 mov x10, #(VVC_MAX_PB_SIZE * 2) cmp width, #8 lsr x11, x6, #32 // weight0 mov w12, w6 // weight1 lsr x13, x7, #32 // offset mov w14, w7 // shift dup v19.8h, w11 neg w14, w14 // so we can use sqshl dup v20.8h, w12 dup v16.4s, w13 dup v22.4s, w14 .if \bit_depth >= 10 // clip pixel mov w6, #((1 << \bit_depth) - 1) dup v17.8h, w6 .endif b.eq 8f b.hi 16f cmp width, #4 b.eq 4f 2: // width == 2 subs height, height, #1 vvc_w_avg_\bit_depth\()_2_4 2 b.ne 2b b 32f 4: // width == 4 subs height, height, #1 vvc_w_avg_\bit_depth\()_2_4 4 b.ne 4b b 32f 8: // width == 8 ld1 {v0.8h}, [src0], x10 ld1 {v2.8h}, [src1], x10 mov v4.16b, v16.16b mov v5.16b, v16.16b smlal v4.4s, v0.4h, v19.4h smlal v4.4s, v2.4h, v20.4h smlal2 v5.4s, v0.8h, v19.8h smlal2 v5.4s, v2.8h, v20.8h sqshl v4.4s, v4.4s, v22.4s sqshl v5.4s, v5.4s, v22.4s sqxtun v4.4h, v4.4s sqxtun2 v4.8h, v5.4s subs height, height, #1 .if \bit_depth == 8 sqxtun v4.8b, v4.8h st1 {v4.8b}, [dst], dst_stride .else umin v4.8h, v4.8h, v17.8h st1 {v4.8h}, [dst], dst_stride .endif b.ne 8b b 32f 16: // width >= 16 mov w6, width mov x7, src0 mov x8, src1 mov x9, dst 17: ldp q0, q1, [x7], #32 ldp q2, q3, [x8], #32 mov v4.16b, v16.16b mov v5.16b, v16.16b mov v6.16b, v16.16b mov v7.16b, v16.16b smlal v4.4s, v0.4h, v19.4h smlal v4.4s, v2.4h, v20.4h smlal2 v5.4s, v0.8h, v19.8h smlal2 v5.4s, v2.8h, v20.8h smlal v6.4s, v1.4h, v19.4h smlal v6.4s, v3.4h, v20.4h smlal2 v7.4s, v1.8h, v19.8h smlal2 v7.4s, v3.8h, v20.8h sqshl v4.4s, v4.4s, v22.4s sqshl v5.4s, v5.4s, v22.4s sqshl v6.4s, v6.4s, v22.4s sqshl v7.4s, v7.4s, v22.4s sqxtun v4.4h, v4.4s sqxtun v6.4h, v6.4s sqxtun2 v4.8h, v5.4s sqxtun2 v6.8h, v7.4s subs w6, w6, #16 .if \bit_depth == 8 sqxtun v4.8b, v4.8h sqxtun2 v4.16b, v6.8h str q4, [x9], #16 .else umin v4.8h, v4.8h, v17.8h umin v6.8h, v6.8h, v17.8h stp q4, q6, [x9], #32 .endif b.ne 17b subs height, height, #1 add src0, src0, x10 add src1, src1, x10 add dst, dst, dst_stride b.ne 16b 32: ret .unreq dst .unreq dst_stride .unreq src0 .unreq src1 .unreq width .unreq height endfunc .endm vvc_w_avg 8 vvc_w_avg 10 vvc_w_avg 12 .macro vvc_avg bit_depth function ff_vvc_avg_\bit_depth\()_neon, export=1 mov x10, #(VVC_MAX_PB_SIZE * 2) movi v16.8h, #0 movi v17.16b, #255 ushr v17.8h, v17.8h, #(16 - \bit_depth) cmp w4, #8 b.gt 16f b.eq 8f cmp w4, #4 b.eq 4f 2: // width == 2 ldr s0, [x2] subs w5, w5, #1 ldr s1, [x3] .if \bit_depth == 8 shadd v0.4h, v0.4h, v1.4h sqrshrun v0.8b, v0.8h, #(15 - 1 - \bit_depth) str h0, [x0] .else shadd v0.4h, v0.4h, v1.4h srshr v0.4h, v0.4h, #(15 - 1 - \bit_depth) smax v0.4h, v0.4h, v16.4h smin v0.4h, v0.4h, v17.4h str s0, [x0] .endif add x2, x2, #(VVC_MAX_PB_SIZE * 2) add x3, x3, #(VVC_MAX_PB_SIZE * 2) add x0, x0, x1 b.ne 2b ret 4: // width == 4 ldr d0, [x2] subs w5, w5, #1 ldr d1, [x3] .if \bit_depth == 8 shadd v0.4h, v0.4h, v1.4h sqrshrun v0.8b, v0.8h, #(15 - 1 - \bit_depth) str s0, [x0] .else shadd v0.4h, v0.4h, v1.4h srshr v0.4h, v0.4h, #(15 - 1 - \bit_depth) smax v0.4h, v0.4h, v16.4h smin v0.4h, v0.4h, v17.4h str d0, [x0] .endif add x2, x2, #(VVC_MAX_PB_SIZE * 2) add x3, x3, #(VVC_MAX_PB_SIZE * 2) add x0, x0, x1 b.ne 4b ret 8: // width == 8 ldr q0, [x2] subs w5, w5, #1 ldr q1, [x3] .if \bit_depth == 8 shadd v0.8h, v0.8h, v1.8h sqrshrun v0.8b, v0.8h, #(15 - 1 - \bit_depth) str d0, [x0] .else shadd v0.8h, v0.8h, v1.8h srshr v0.8h, v0.8h, #(15 - 1 - \bit_depth) smax v0.8h, v0.8h, v16.8h smin v0.8h, v0.8h, v17.8h str q0, [x0] .endif add x2, x2, #(VVC_MAX_PB_SIZE * 2) add x3, x3, #(VVC_MAX_PB_SIZE * 2) add x0, x0, x1 b.ne 8b ret 16: // width >= 16 .if \bit_depth == 8 sub x1, x1, w4, sxtw .else sub x1, x1, w4, sxtw #1 .endif sub x10, x10, w4, sxtw #1 3: mov w6, w4 // width 1: ldp q0, q1, [x2], #32 subs w6, w6, #16 ldp q2, q3, [x3], #32 .if \bit_depth == 8 shadd v4.8h, v0.8h, v2.8h shadd v5.8h, v1.8h, v3.8h sqrshrun v0.8b, v4.8h, #6 sqrshrun2 v0.16b, v5.8h, #6 st1 {v0.16b}, [x0], #16 .else shadd v4.8h, v0.8h, v2.8h shadd v5.8h, v1.8h, v3.8h srshr v0.8h, v4.8h, #(15 - 1 - \bit_depth) srshr v1.8h, v5.8h, #(15 - 1 - \bit_depth) smax v0.8h, v0.8h, v16.8h smax v1.8h, v1.8h, v16.8h smin v0.8h, v0.8h, v17.8h smin v1.8h, v1.8h, v17.8h stp q0, q1, [x0], #32 .endif b.ne 1b subs w5, w5, #1 add x2, x2, x10 add x3, x3, x10 add x0, x0, x1 b.ne 3b ret endfunc .endm vvc_avg 8 vvc_avg 10 vvc_avg 12 /* x0: int16_t *dst * x1: const uint8_t *_src * x2: ptrdiff_t _src_stride * w3: int height * x4: intptr_t mx * x5: intptr_t my * w6: int width */ function ff_vvc_dmvr_8_neon, export=1 dst .req x0 src .req x1 src_stride .req x2 height .req w3 mx .req x4 my .req x5 width .req w6 sxtw x6, w6 mov x7, #(VVC_MAX_PB_SIZE * 2 + 8) cmp width, #16 sub src_stride, src_stride, x6 cset w15, gt // width > 16 movi v16.8h, #2 // DMVR_SHIFT sub x7, x7, x6, lsl #1 1: cbz w15, 2f ldr q0, [src], #16 ushll v1.8h, v0.8b, #2 ushll2 v2.8h, v0.16b, #2 stp q1, q2, [dst], #32 b 3f 2: ldr d0, [src], #8 ushll v1.8h, v0.8b, #2 str q1, [dst], #16 3: subs height, height, #1 ldr s3, [src], #4 ushll v4.8h, v3.8b, #2 st1 {v4.4h}, [dst], x7 add src, src, src_stride b.ne 1b ret endfunc function ff_vvc_dmvr_12_neon, export=1 sxtw x6, w6 mov x7, #(VVC_MAX_PB_SIZE * 2 + 8) cmp width, #16 sub src_stride, src_stride, x6, lsl #1 cset w15, gt // width > 16 sub x7, x7, x6, lsl #1 1: cbz w15, 2f ldp q0, q1, [src], #32 urshr v0.8h, v0.8h, #2 urshr v1.8h, v1.8h, #2 stp q0, q1, [dst], #32 b 3f 2: ldr q0, [src], #16 urshr v0.8h, v0.8h, #2 str q0, [dst], #16 3: subs height, height, #1 ldr d0, [src], #8 urshr v0.4h, v0.4h, #2 st1 {v0.4h}, [dst], x7 add src, src, src_stride b.ne 1b ret endfunc function ff_vvc_dmvr_v_8_neon, export=1 movrel x7, X(ff_vvc_inter_luma_dmvr_filters) add x7, x7, x5, lsl #1 ld2r {v0.16b, v1.16b}, [x7] tbz w6, #4, 12f ldr s16, [x1, #16] ld1 {v2.16b}, [x1], x2 20: ldr s17, [x1, #16] umull v4.8h, v0.8b, v2.8b umull2 v5.8h, v0.16b, v2.16b ld1 {v3.16b}, [x1], x2 umull v16.8h, v0.8b, v16.8b umull v6.8h, v1.8b, v3.8b umull2 v7.8h, v1.16b, v3.16b add v4.8h, v4.8h, v6.8h umull v18.8h, v1.8b, v17.8b add v5.8h, v5.8h, v7.8h urshr v4.8h, v4.8h, #2 add v19.4h, v16.4h, v18.4h urshr v5.8h, v5.8h, #2 urshr v19.4h, v19.4h, #2 st1 {v4.8h, v5.8h}, [x0], #32 subs w3, w3, #1 mov v2.16b, v3.16b st1 {v19.4h}, [x0], #8 mov v16.16b, v17.16b add x0, x0, #(VVC_MAX_PB_SIZE * 2 - 32 - 8) b.ne 20b ret 12: ldr s16, [x1, #8] ld1 {v2.8b}, [x1], x2 2: ldr s17, [x1, #8] umull v4.8h, v0.8b, v2.8b ld1 {v3.8b}, [x1], x2 umull v16.8h, v0.8b, v16.8b umull v6.8h, v1.8b, v3.8b add v4.8h, v4.8h, v6.8h umull v18.8h, v1.8b, v17.8b srshr v4.8h, v4.8h, #2 add v19.4h, v16.4h, v18.4h srshr v19.4h, v19.4h, #2 st1 {v4.8h}, [x0], #16 subs w3, w3, #1 mov v2.16b, v3.16b st1 {v19.4h}, [x0], #8 mov v16.16b, v17.16b add x0, x0, #(VVC_MAX_PB_SIZE * 2 - 16 - 8) b.ne 2b ret endfunc function ff_vvc_dmvr_h_8_neon, export=1 movrel x7, X(ff_vvc_inter_luma_dmvr_filters) add x7, x7, x4, lsl #1 ld2r {v0.16b, v1.16b}, [x7] tbz w6, #4, 12f 20: ldur q3, [x1, #1] ldr q2, [x1] umull v4.8h, v0.8b, v2.8b umull2 v5.8h, v0.16b, v2.16b ldur s17, [x1, #17] umull v6.8h, v1.8b, v3.8b ldr s16, [x1, #16] umull2 v7.8h, v1.16b, v3.16b add v4.8h, v4.8h, v6.8h umull v17.8h, v1.8b, v17.8b add v5.8h, v5.8h, v7.8h umull v16.8h, v0.8b, v16.8b srshr v4.8h, v4.8h, #2 add v16.4h, v16.4h, v17.4h srshr v5.8h, v5.8h, #2 srshr v16.4h, v16.4h, #2 st1 {v4.8h, v5.8h}, [x0], #32 subs w3, w3, #1 st1 {v16.4h}, [x0], #8 add x1, x1, x2 add x0, x0, #(VVC_MAX_PB_SIZE * 2 - 32 - 8) b.ne 20b ret 12: ldur d3, [x1, #1] ldr d2, [x1] umull v4.8h, v0.8b, v2.8b ldur s17, [x1, #9] umull v6.8h, v1.8b, v3.8b ldr s16, [x1, #8] add v4.8h, v4.8h, v6.8h umull v17.8h, v1.8b, v17.8b umull v16.8h, v0.8b, v16.8b srshr v4.8h, v4.8h, #2 add v16.4h, v16.4h, v17.4h srshr v16.4h, v16.4h, #2 st1 {v4.8h}, [x0], #16 subs w3, w3, #1 st1 {v16.4h}, [x0], #8 add x1, x1, x2 add x0, x0, #(VVC_MAX_PB_SIZE * 2 - 16 - 8) b.ne 12b ret endfunc .macro vvc_dmvr_h_10 bit_depth function ff_vvc_dmvr_h_\bit_depth\()_neon, export=1 movrel x7, X(ff_vvc_inter_luma_dmvr_filters) add x7, x7, x4, lsl #1 ld2r {v0.16b, v1.16b}, [x7] uxtl v0.8h, v0.8b uxtl v1.8h, v1.8b tbz w6, #4, 12f 20: ldur q3, [x1, #2] ldr q2, [x1] ldr q22, [x1, #16] mul v4.8h, v0.8h, v2.8h mul v6.8h, v1.8h, v3.8h ldur q23, [x1, #18] mul v5.8h, v0.8h, v22.8h ldur d17, [x1, #34] mul v7.8h, v1.8h, v23.8h uhadd v4.8h, v4.8h, v6.8h ldr d16, [x1, #32] uhadd v5.8h, v5.8h, v7.8h mul v17.4h, v1.4h, v17.4h mul v16.4h, v0.4h, v16.4h urshr v4.8h, v4.8h, #(\bit_depth - 6 - 1) urshr v5.8h, v5.8h, #(\bit_depth - 6 - 1) uhadd v16.4h, v16.4h, v17.4h urshr v16.4h, v16.4h, #(\bit_depth - 6 - 1) st1 {v4.8h, v5.8h}, [x0], #32 subs w3, w3, #1 st1 {v16.4h}, [x0], #8 add x1, x1, x2 add x0, x0, #(VVC_MAX_PB_SIZE * 2 - 32 - 8) b.ne 20b ret 12: ldur q3, [x1, #2] ldr q2, [x1] mul v4.8h, v0.8h, v2.8h ldur d17, [x1, #18] mul v6.8h, v1.8h, v3.8h ldr d16, [x1, #16] uhadd v4.8h, v4.8h, v6.8h mul v17.4h, v1.4h, v17.4h mul v16.4h, v0.4h, v16.4h urshr v4.8h, v4.8h, #(\bit_depth - 6 - 1) uhadd v16.4h, v16.4h, v17.4h urshr v16.4h, v16.4h, #(\bit_depth - 6 - 1) st1 {v4.8h}, [x0], #16 subs w3, w3, #1 st1 {v16.4h}, [x0], #8 add x1, x1, x2 add x0, x0, #(VVC_MAX_PB_SIZE * 2 - 16 - 8) b.ne 12b ret endfunc .endm vvc_dmvr_h_10 10 vvc_dmvr_h_10 12 function ff_vvc_dmvr_hv_8_neon, export=1 tmp0 .req x7 tmp1 .req x8 sub sp, sp, #(VVC_MAX_PB_SIZE * 4) movrel x9, X(ff_vvc_inter_luma_dmvr_filters) add x12, x9, mx, lsl #1 mov tmp0, sp add tmp1, tmp0, #(VVC_MAX_PB_SIZE * 2) // We know the value are positive ld2r {v0.16b, v1.16b}, [x12] add x12, x9, my, lsl #1 ldrb w10, [x12] ldrb w11, [x12, #1] sxtw x6, w6 dup v2.8h, w10 // filter_y[0] dup v3.8h, w11 // filter_y[1] // Valid value for width can only be 8 + 4, 16 + 4 cmp width, #16 mov w10, #0 // start filter_y or not add height, height, #1 sub dst, dst, #(VVC_MAX_PB_SIZE * 2) sub src_stride, src_stride, x6 cset w15, gt // width > 16 1: mov x12, tmp0 mov x13, tmp1 mov x14, dst cbz w15, 2f // width > 16 ldur q5, [src, #1] ldr q4, [src], #16 umull v6.8h, v4.8b, v0.8b umull2 v16.8h, v4.16b, v0.16b umlal v6.8h, v5.8b, v1.8b umlal2 v16.8h, v5.16b, v1.16b urshr v6.8h, v6.8h, #(8 - 6) urshr v7.8h, v16.8h, #(8 - 6) stp q6, q7, [x13], #32 cbz w10, 3f ldp q16, q17, [x12], #32 mul v16.8h, v16.8h, v2.8h mul v17.8h, v17.8h, v2.8h mla v16.8h, v6.8h, v3.8h mla v17.8h, v7.8h, v3.8h urshr v16.8h, v16.8h, #4 urshr v17.8h, v17.8h, #4 stp q16, q17, [x14], #32 b 3f 2: // width > 8 ldur d5, [src, #1] ldr d4, [src], #8 umull v6.8h, v4.8b, v0.8b umlal v6.8h, v5.8b, v1.8b urshr v6.8h, v6.8h, #(8 - 6) str q6, [x13], #16 cbz w10, 3f ldr q16, [x12], #16 mul v16.8h, v16.8h, v2.8h mla v16.8h, v6.8h, v3.8h urshr v16.8h, v16.8h, #4 str q16, [x14], #16 3: ldur s5, [src, #1] ldr s4, [src], #4 umull v6.8h, v4.8b, v0.8b umlal v6.8h, v5.8b, v1.8b urshr v6.4h, v6.4h, #(8 - 6) str d6, [x13], #8 cbz w10, 4f ldr d16, [x12], #8 mul v16.4h, v16.4h, v2.4h mla v16.4h, v6.4h, v3.4h urshr v16.4h, v16.4h, #4 str d16, [x14], #8 4: subs height, height, #1 mov w10, #1 add src, src, src_stride add dst, dst, #(VVC_MAX_PB_SIZE * 2) eor tmp0, tmp0, tmp1 eor tmp1, tmp0, tmp1 eor tmp0, tmp0, tmp1 b.ne 1b add sp, sp, #(VVC_MAX_PB_SIZE * 4) ret endfunc function ff_vvc_dmvr_hv_12_neon, export=1 mvni v29.4s, #(12 - 6 - 1) b 0f endfunc function ff_vvc_dmvr_hv_10_neon, export=1 mvni v29.4s, #(10 - 6 - 1) 0: sub sp, sp, #(VVC_MAX_PB_SIZE * 4) movrel x9, X(ff_vvc_inter_luma_dmvr_filters) add x12, x9, mx, lsl #1 ldrb w10, [x12] ldrb w11, [x12, #1] mov tmp0, sp add tmp1, tmp0, #(VVC_MAX_PB_SIZE * 2) // We know the value are positive dup v0.8h, w10 // filter_x[0] dup v1.8h, w11 // filter_x[1] add x12, x9, my, lsl #1 ldrb w10, [x12] ldrb w11, [x12, #1] dup v2.8h, w10 // filter_y[0] dup v3.8h, w11 // filter_y[1] // Valid value for width can only be 8 + 4, 16 + 4 cmp width, #16 mov w10, #0 // start filter_y or not add height, height, #1 sub dst, dst, #(VVC_MAX_PB_SIZE * 2) sub src_stride, src_stride, w6, sxtw #1 cset w15, gt // width > 16 1: mov x12, tmp0 mov x13, tmp1 mov x14, dst cbz w15, 2f // width > 16 add x16, src, #2 ldp q6, q16, [src], #32 ldp q7, q17, [x16] umull v4.4s, v6.4h, v0.4h umull2 v5.4s, v6.8h, v0.8h umull v18.4s, v16.4h, v0.4h umull2 v19.4s, v16.8h, v0.8h umlal v4.4s, v7.4h, v1.4h umlal2 v5.4s, v7.8h, v1.8h umlal v18.4s, v17.4h, v1.4h umlal2 v19.4s, v17.8h, v1.8h urshl v4.4s, v4.4s, v29.4s urshl v5.4s, v5.4s, v29.4s urshl v18.4s, v18.4s, v29.4s urshl v19.4s, v19.4s, v29.4s uqxtn v6.4h, v4.4s uqxtn2 v6.8h, v5.4s uqxtn v7.4h, v18.4s uqxtn2 v7.8h, v19.4s stp q6, q7, [x13], #32 cbz w10, 3f ldp q4, q5, [x12], #32 umull v17.4s, v4.4h, v2.4h umull2 v18.4s, v4.8h, v2.8h umull v19.4s, v5.4h, v2.4h umull2 v20.4s, v5.8h, v2.8h umlal v17.4s, v6.4h, v3.4h umlal2 v18.4s, v6.8h, v3.8h umlal v19.4s, v7.4h, v3.4h umlal2 v20.4s, v7.8h, v3.8h uqrshrn v6.4h, v17.4s, #4 uqrshrn2 v6.8h, v18.4s, #4 uqrshrn v7.4h, v19.4s, #4 uqrshrn2 v7.8h, v20.4s, #4 stp q6, q7, [x14], #32 b 3f 2: // width > 8 ldur q7, [src, #2] ldr q6, [src], #16 umull v4.4s, v6.4h, v0.4h umull2 v5.4s, v6.8h, v0.8h umlal v4.4s, v7.4h, v1.4h umlal2 v5.4s, v7.8h, v1.8h urshl v4.4s, v4.4s, v29.4s urshl v5.4s, v5.4s, v29.4s uqxtn v6.4h, v4.4s uqxtn2 v6.8h, v5.4s str q6, [x13], #16 cbz w10, 3f ldr q16, [x12], #16 umull v17.4s, v16.4h, v2.4h umull2 v18.4s, v16.8h, v2.8h umlal v17.4s, v6.4h, v3.4h umlal2 v18.4s, v6.8h, v3.8h urshr v17.4s, v17.4s, #4 urshr v18.4s, v18.4s, #4 uqxtn v16.4h, v17.4s uqxtn2 v16.8h, v18.4s str q16, [x14], #16 3: ldur d7, [src, #2] ldr d6, [src], #8 umull v4.4s, v7.4h, v1.4h umlal v4.4s, v6.4h, v0.4h urshl v4.4s, v4.4s, v29.4s uqxtn v6.4h, v4.4s str d6, [x13], #8 cbz w10, 4f ldr d16, [x12], #8 umull v17.4s, v16.4h, v2.4h umlal v17.4s, v6.4h, v3.4h urshr v17.4s, v17.4s, #4 uqxtn v16.4h, v17.4s str d16, [x14], #8 4: subs height, height, #1 mov w10, #1 add src, src, src_stride add dst, dst, #(VVC_MAX_PB_SIZE * 2) eor tmp0, tmp0, tmp1 eor tmp1, tmp0, tmp1 eor tmp0, tmp0, tmp1 b.ne 1b add sp, sp, #(VVC_MAX_PB_SIZE * 4) ret .unreq dst .unreq src .unreq src_stride .unreq height .unreq mx .unreq my .unreq width .unreq tmp0 .unreq tmp1 endfunc function ff_vvc_prof_grad_filter_8x_neon, export=1 gh .req x0 gv .req x1 gstride .req x2 src .req x3 src_stride .req x4 width .req w5 height .req w6 lsl src_stride, src_stride, #1 neg x7, src_stride 1: mov x10, src mov w11, width mov x12, gh mov x13, gv 2: ldur q0, [x10, #2] ldur q1, [x10, #-2] subs w11, w11, #8 ldr q2, [x10, src_stride] ldr q3, [x10, x7] sshr v0.8h, v0.8h, #6 sshr v1.8h, v1.8h, #6 sshr v2.8h, v2.8h, #6 sshr v3.8h, v3.8h, #6 sub v0.8h, v0.8h, v1.8h sub v2.8h, v2.8h, v3.8h st1 {v0.8h}, [x12], #16 st1 {v2.8h}, [x13], #16 add x10, x10, #16 b.ne 2b subs height, height, #1 add gh, gh, gstride, lsl #1 add gv, gv, gstride, lsl #1 add src, src, src_stride b.ne 1b ret .unreq gh .unreq gv .unreq gstride .unreq src .unreq src_stride .unreq width .unreq height endfunc function vvc_bdof_grad_filter_8x_neon, export=0 gh0 .req x0 gh1 .req x1 gv0 .req x2 gv1 .req x3 src0 .req x4 src1 .req x5 width .req w6 height .req w7 tbnz w6, #4, 16f 8: ldur q0, [src0, #2] ldur q1, [src0, #-2] ldr q2, [src0, #(VVC_MAX_PB_SIZE << 1)] ldr q3, [src0, #-(VVC_MAX_PB_SIZE << 1)] sshr v0.8h, v0.8h, #6 sshr v1.8h, v1.8h, #6 ldur q4, [src1, #2] ldur q5, [src1, #-2] sshr v2.8h, v2.8h, #6 sshr v3.8h, v3.8h, #6 ldr q6, [src1, #(VVC_MAX_PB_SIZE << 1)] ldr q7, [src1, #-(VVC_MAX_PB_SIZE << 1)] // results of gradient_h0 sub v0.8h, v0.8h, v1.8h // results of gradient_v0 sub v2.8h, v2.8h, v3.8h sshr v4.8h, v4.8h, #6 sshr v5.8h, v5.8h, #6 sshr v6.8h, v6.8h, #6 sshr v7.8h, v7.8h, #6 // results of gradient_h1 sub v4.8h, v4.8h, v5.8h // results of gradient_v1 sub v6.8h, v6.8h, v7.8h // (gradient_h0 + gradient_h1) >> 1 shadd v1.8h, v0.8h, v4.8h // gradient_h0 - gradient_h1 sub v5.8h, v0.8h, v4.8h // (gradient_v0 + gradient_v1) >> 1 shadd v3.8h, v2.8h, v6.8h // gradient_v0 - gradient_v1 sub v7.8h, v2.8h, v6.8h st1 {v1.8h}, [gh0] st1 {v5.8h}, [gh1] st1 {v3.8h}, [gv0] st1 {v7.8h}, [gv1] subs height, height, #1 add gh0, gh0, #(BDOF_BLOCK_SIZE << 1) add gv0, gv0, #(BDOF_BLOCK_SIZE << 1) add src0, src0, #(VVC_MAX_PB_SIZE << 1) add gh1, gh1, #(BDOF_BLOCK_SIZE << 1) add gv1, gv1, #(BDOF_BLOCK_SIZE << 1) add src1, src1, #(VVC_MAX_PB_SIZE << 1) b.ne 8b ret 16: ldur q0, [src0, #2] ldur q1, [src0, #18] ldur q16, [src0, #-2] sshr v0.8h, v0.8h, #6 ldur q17, [src0, #14] sshr v1.8h, v1.8h, #6 ldp q18, q19, [src0, #-(VVC_MAX_PB_SIZE << 1)] sshr v16.8h, v16.8h, #6 ldp q2, q3, [src0, #(VVC_MAX_PB_SIZE << 1)]! ldur q20, [src1, #2] sshr v17.8h, v17.8h, #6 ldur q21, [src1, #18] sshr v2.8h, v2.8h, #6 ldur q22, [src1, #-2] sshr v3.8h, v3.8h, #6 ldur q23, [src1, #14] sshr v18.8h, v18.8h, #6 ldp q26, q27, [src1, #-(VVC_MAX_PB_SIZE << 1)] sshr v19.8h, v19.8h, #6 ldp q24, q25, [src1, #(VVC_MAX_PB_SIZE << 1)]! // results of gradient_h0 sub v0.8h, v0.8h, v16.8h sub v1.8h, v1.8h, v17.8h // results of gradient_v0 sub v2.8h, v2.8h, v18.8h sub v3.8h, v3.8h, v19.8h sshr v20.8h, v20.8h, #6 sshr v21.8h, v21.8h, #6 sshr v22.8h, v22.8h, #6 sshr v23.8h, v23.8h, #6 // results of gradient_h1 sub v20.8h, v20.8h, v22.8h sub v21.8h, v21.8h, v23.8h sshr v24.8h, v24.8h, #6 sshr v25.8h, v25.8h, #6 // gradient_h0 - gradient_h1 sub v22.8h, v0.8h, v20.8h sub v23.8h, v1.8h, v21.8h // (gradient_h0 + gradient_h1) >> 1 shadd v16.8h, v0.8h, v20.8h shadd v17.8h, v1.8h, v21.8h st1 {v22.8h, v23.8h}, [gh1], #32 sshr v26.8h, v26.8h, #6 sshr v27.8h, v27.8h, #6 st1 {v16.8h, v17.8h}, [gh0], #32 // results of gradient_v1 sub v24.8h, v24.8h, v26.8h sub v25.8h, v25.8h, v27.8h // (gradient_v0 + gradient_v1) >> 1 shadd v18.8h, v2.8h, v24.8h shadd v19.8h, v3.8h, v25.8h // gradient_v0 - gradient_v1 sub v26.8h, v2.8h, v24.8h sub v27.8h, v3.8h, v25.8h st1 {v18.8h,v19.8h}, [gv0], #32 subs height, height, #1 st1 {v26.8h,v27.8h}, [gv1], #32 b.ne 16b ret .unreq gh0 .unreq gh1 .unreq gv0 .unreq gv1 .unreq src0 .unreq src1 .unreq width .unreq height endfunc .macro vvc_apply_bdof_block_8x bit_depth dst .req x0 dst_stride .req x1 src0 .req x2 src1 .req x3 gh .req x4 gv .req x5 vx .req x6 vy .req x7 ldr w8, [sp] mov x12, #(BDOF_BLOCK_SIZE * 2) mov x14, #(VVC_MAX_PB_SIZE * 2) .if \bit_depth >= 10 // clip pixel mov w15, #((1 << \bit_depth) - 1) dup v19.8h, w15 .endif 0: ldr s0, [vx], #(2 * BDOF_MIN_BLOCK_SIZE) ldr s1, [vy], #(2 * BDOF_MIN_BLOCK_SIZE) mov w13, #(BDOF_MIN_BLOCK_SIZE) 1: ld1 {v5.8h}, [src0], x14 ld1 {v6.8h}, [src1], x14 saddl v17.4s, v5.4h, v6.4h ld1 {v4.8h}, [gv], x12 saddl2 v16.4s, v5.8h, v6.8h ld1 {v2.8h}, [gh], x12 smlal v17.4s, v4.4h, v1.h[0] smlal2 v16.4s, v4.8h, v1.h[1] smlal v17.4s, v2.4h, v0.h[0] smlal2 v16.4s, v2.8h, v0.h[1] sqrshrun v5.4h, v17.4s, #(15 - \bit_depth) sqrshrun2 v5.8h, v16.4s, #(15 - \bit_depth) subs w13, w13, #1 .if \bit_depth == 8 sqxtun v5.8b, v5.8h st1 {v5.8b}, [dst], dst_stride .else smin v5.8h, v5.8h, v19.8h st1 {v5.8h}, [dst], dst_stride .endif b.ne 1b subs w8, w8, #(BDOF_MIN_BLOCK_SIZE) b.ne 0b ret .unreq dst .unreq dst_stride .unreq src0 .unreq src1 .unreq gh .unreq gv .unreq vx .unreq vy .endm function vvc_apply_bdof_block_8x_8_neon, export=0 vvc_apply_bdof_block_8x 8 endfunc function vvc_apply_bdof_block_8x_10_neon, export=0 vvc_apply_bdof_block_8x 10 endfunc function vvc_apply_bdof_block_8x_12_neon, export=0 vvc_apply_bdof_block_8x 12 endfunc .macro vvc_apply_bdof_block_16x bit_depth dst .req x0 dst_stride .req x1 src0 .req x2 src1 .req x3 gh .req x4 gv .req x5 vx .req x6 vy .req x7 ldr w8, [sp] movi v7.4s, #(1 << (14 - \bit_depth)) .if \bit_depth >= 10 // clip pixel mov w15, #((1 << \bit_depth) - 1) movi v18.8h, #0 dup v19.8h, w15 .endif 0: ld1r {v0.8h}, [vx], #2 ld1r {v1.8h}, [vy], #2 ld1r {v2.8h}, [vx], #2 ld1r {v3.8h}, [vy], #2 mov w13, #(BDOF_MIN_BLOCK_SIZE) ld1r {v20.8h}, [vx], #2 ld1r {v21.8h}, [vy], #2 ld1r {v22.8h}, [vx], #2 ld1r {v23.8h}, [vy], #2 ins v0.d[1], v2.d[1] ins v1.d[1], v3.d[1] ins v20.d[1], v22.d[1] ins v21.d[1], v23.d[1] 1: ldp q2, q22, [gh], #(BDOF_BLOCK_SIZE * 2) ldp q4, q24, [gv], #(BDOF_BLOCK_SIZE * 2) smull v3.4s, v0.4h, v2.4h smull2 v16.4s, v0.8h, v2.8h smlal v3.4s, v1.4h, v4.4h smlal2 v16.4s, v1.8h, v4.8h ldp q5, q25, [src0], #(VVC_MAX_PB_SIZE * 2) ldp q6, q26, [src1], #(VVC_MAX_PB_SIZE * 2) smull v23.4s, v20.4h, v22.4h smull2 v27.4s, v20.8h, v22.8h smlal v23.4s, v21.4h, v24.4h smlal2 v27.4s, v21.8h, v24.8h saddl v2.4s, v5.4h, v6.4h add v2.4s, v2.4s, v7.4s add v2.4s, v2.4s, v3.4s saddl2 v4.4s, v5.8h, v6.8h add v4.4s, v4.4s, v7.4s add v4.4s, v4.4s, v16.4s saddl v22.4s, v25.4h, v26.4h add v22.4s, v22.4s, v7.4s add v22.4s, v22.4s, v23.4s saddl2 v24.4s, v25.8h, v26.8h add v24.4s, v24.4s, v7.4s add v24.4s, v24.4s, v27.4s sqshrn v5.4h, v2.4s, #(15 - \bit_depth) sqshrn2 v5.8h, v4.4s, #(15 - \bit_depth) sqshrn v25.4h, v22.4s, #(15 - \bit_depth) sqshrn2 v25.8h, v24.4s, #(15 - \bit_depth) subs w13, w13, #1 .if \bit_depth == 8 sqxtun v5.8b, v5.8h sqxtun2 v5.16b, v25.8h str q5, [dst] .else smin v5.8h, v5.8h, v19.8h smax v5.8h, v5.8h, v18.8h smin v25.8h, v25.8h, v19.8h smax v25.8h, v25.8h, v18.8h stp q5, q25, [dst] .endif add dst, dst, dst_stride b.ne 1b subs w8, w8, #(BDOF_MIN_BLOCK_SIZE) b.ne 0b ret .unreq dst .unreq dst_stride .unreq src0 .unreq src1 .unreq gh .unreq gv .unreq vx .unreq vy .endm function vvc_apply_bdof_block_16x_8_neon, export=0 vvc_apply_bdof_block_16x 8 endfunc function vvc_apply_bdof_block_16x_10_neon, export=0 vvc_apply_bdof_block_16x 10 endfunc function vvc_apply_bdof_block_16x_12_neon, export=0 vvc_apply_bdof_block_16x 12 endfunc const bdof_vx_vy_8x_tbl .byte 0, 1, 16, 16, 16, 16, 8, 9 .byte 6, 7, 16, 16, 16, 16, 14, 15 endconst const bdof_vx_vy_16x_tbl .byte 0, 1, 64, 64, 64, 64, 8, 9 .byte 6, 7, 64, 64, 64, 64, 16, 17 .byte 14, 15, 64, 64, 64, 64, 24, 25 .byte 22, 23, 64, 64, 64, 64, 30, 31 endconst // line(-1), line0, line1, line2, line3, line4 // line3 and line4 becomes line(-1) and line0 in the next block. .macro bdof_vx_vy_8x_save_line tmp0, tmp1, tmp2, tmp3, tmp4 mov \tmp0\().16b, v28.16b mov \tmp1\().16b, v29.16b mov \tmp2\().16b, v30.16b mov \tmp3\().16b, v31.16b mov \tmp4\().16b, v8.16b .endm .macro bdof_vx_vy_8x_add_line tmp0, tmp1, tmp2, tmp3, tmp4 add v25.4s, v25.4s, \tmp0\().4s add v27.4s, v27.4s, \tmp1\().4s add v23.4s, v23.4s, \tmp2\().4s sub v26.4s, v26.4s, \tmp3\().4s sub v24.4s, v24.4s, \tmp4\().4s .endm .macro bdof_vx_vy_8x_padding_left_right src, tmp0, tmp1, dst tbl \tmp0\().16b, { \src\().16b }, v0.16b saddl \tmp1\().4s, \tmp0\().4h, \src\().4h saddl2 \dst\().4s, \tmp0\().8h, \src\().8h addp \dst\().4s, \tmp1\().4s, \dst\().4s .endm .macro bdof_vx_vy_sign src, tmp0, tmp1, dst cmlt \tmp0\().8h, \src\().8h, #0 cmgt \tmp1\().8h, \src\().8h, #0 sub \dst\().8h, \tmp0\().8h, \tmp1\().8h .endm .macro bdof_vx_vy_clip_mask src, max, min, mask, dst smin \src\().4s, \src\().4s, \max\().4s smax \src\().4s, \src\().4s, \min\().4s cmgt \mask\().4s, \mask\().4s, #0 and \dst\().16b, \src\().16b, \mask\().16b .endm .macro bdof_vx_vy_16x_save_line tmp0, tmp1, tmp2, tmp3, tmp4 mov \tmp0\().16b, v29.16b mov \tmp1\().16b, v30.16b mov \tmp2\().16b, v31.16b mov \tmp3\().16b, v8.16b mov \tmp4\().16b, v9.16b .endm .macro bdof_vx_vy_16x_add_line tmp0, tmp1, tmp2, tmp3, tmp4 add v25.4s, v25.4s, \tmp0\().4s add v24.4s, v24.4s, \tmp1\().4s add v26.4s, v26.4s, \tmp2\().4s sub v28.4s, v28.4s, \tmp3\().4s sub v27.4s, v27.4s, \tmp4\().4s .endm .macro bdof_vx_vy_16x_padding_left_right src0, src1, tmp0, tmp1, tmp2, dst tbl \tmp0\().16b, {\src0\().16b, \src1\().16b}, v0.16b tbl v2.16b, {\src0\().16b, \src1\().16b}, v1.16b saddl \tmp1\().4s, \tmp0\().4h, \src0\().4h saddl \tmp2\().4s, v2.4h, \src1\().4h saddl2 \tmp0\().4s, \tmp0\().8h, \src0\().8h saddl2 \dst\().4s, v2.8h, \src1\().8h addp \tmp0\().4s, \tmp1\().4s, \tmp0\().4s addp \dst\().4s, \tmp2\().4s, \dst\().4s addp \dst\().4s, \tmp0\().4s, \dst\().4s .endm /* * Line tricks: * We need 6 lines of information, from 4N-1, 4N, 4N+1 to 4N+4. 4N-1 * and 4N+0 are processed in the last group, so they can be reused. * * (4N-1) [xxxxxxxxxxxxx] <--- reuse * (4N) [xxxxxxxxxxxxx] <--- reuse * (4N+1) [xxxxxxxxxxxxx] * (4N+2) [xxxxxxxxxxxxx] * (4N+3) [xxxxxxxxxxxxx] ---> save for reuse * (4N+4) [xxxxxxxxxxxxx] ---> save for reuse * * Special case: * 1. Line -1 needs to duplicate line 0. * 2. Last line +1 needs to duplicate the last line. * * --------------------------------------------------------------------- * Pixel tricks: * * [C-1, C0, C1, C2, ... C16] * * For each line, we need to sum parameters for 4 * 6 pixels: * - C-1 + C0 + C1 + C2 + C3 + C4 * - C3 + C4 + C5 + C6 + C7 + C8 * - C7 + C8 + C9 + C10 + C11 + C12 * - C11 + C12 + C13 + C14 + C15 + C16 * * C-1 is C0, C16 is C15, so we can do: * * [C0, C1, C2, C3, | C4, C5, C6, C7, | C8, ... C15] * + | + | * [C0, 0, 0, C4, | C3, 0, 0, C8, | C7, ... C15] * * 8x is similar. * ---------------------------------------------------------------------- * x0: const int16_t *_src0, * x1: const int16_t *_src1, * x2: const int16_t *gradient_h, * x3: const int16_t *gradient_v, * x4: int16_t vx[16], * x5: int16_t vy[16], * w6: int block_h */ function vvc_derive_bdof_vx_vy_8x_neon, export=0 stp d11, d10, [sp, #-0x20]! stp d9, d8, [sp, #0x10] movrel x11, bdof_vx_vy_8x_tbl ldr q0, [x11] // table mvni v2.4s, #30 // -31, for log2 movi v3.4s, #15 // clip to 15 mvni v4.4s, #14 // clip to -15 mov w11, #0x8 mov w12, w6 // y = block_h b 4f 1: // save line4 results bdof_vx_vy_8x_save_line v5, v6, v7, v16, v17 2: addp v25.4s, v25.4s, v25.4s addp v27.4s, v27.4s, v27.4s addp v26.4s, v26.4s, v26.4s addp v23.4s, v23.4s, v23.4s addp v24.4s, v24.4s, v24.4s clz v28.4s, v25.4s add v28.4s, v28.4s, v2.4s // log2 shl v26.4s, v26.4s, #0x2 sshl v26.4s, v26.4s, v28.4s bdof_vx_vy_clip_mask v26, v3, v4, v25, v25 sqxtn v26.4h, v25.4s st1 {v26.s}[0], [x4], x11 subs x12, x12, #(BDOF_MIN_BLOCK_SIZE) clz v26.4s, v27.4s add v26.4s, v26.4s, v2.4s shl v24.4s, v24.4s, #0x2 mul v23.4s, v25.4s, v23.4s sshr v23.4s, v23.4s, #0x1 sub v23.4s, v24.4s, v23.4s sshl v23.4s, v23.4s, v26.4s bdof_vx_vy_clip_mask v23, v3, v4, v27, v23 sqxtn v23.4h, v23.4s st1 {v23.s}[0], [x5], x11 b.eq 16f 4: mov x15, #0x0 // dy, inner loop movi v25.2d, #0 movi v27.2d, #0 movi v23.2d, #0 movi v26.2d, #0 movi v24.2d, #0 b 8f 5: // add line(-1) and line0 from previous results bdof_vx_vy_8x_add_line v18, v19, v20, v21, v22 bdof_vx_vy_8x_add_line v5, v6, v7, v16, v17 add x15, x15, #1 8: cmp w12, w6 b.hs 9f // y < block_h && dy == 0, reuse previous results cbz x15, 5b 9: ldr q28, [x0] // src0 ldr q29, [x1] // src1 ldr q30, [x2], #(BDOF_BLOCK_SIZE * 2) // (gh0 + gh1) >> 1 ldr q31, [x3], #(BDOF_BLOCK_SIZE * 2) // (gv0 + gv1) >> 1 add x0, x0, #(VVC_MAX_PB_SIZE * 2) add x1, x1, #(VVC_MAX_PB_SIZE * 2) sshr v28.8h, v28.8h, #0x4 sshr v29.8h, v29.8h, #0x4 sub v8.8h, v28.8h, v29.8h // diff abs v28.8h, v30.8h abs v29.8h, v31.8h bdof_vx_vy_8x_padding_left_right v28, v9, v10, v28 bdof_vx_vy_8x_padding_left_right v29, v9, v10, v29 bdof_vx_vy_sign v30, v9, v10, v9 bdof_vx_vy_sign v31, v10, v31, v31 mul v30.8h, v31.8h, v30.8h mul v9.8h, v9.8h, v8.8h mul v8.8h, v31.8h, v8.8h bdof_vx_vy_8x_padding_left_right v30, v31, v10, v30 bdof_vx_vy_8x_padding_left_right v9, v31, v10, v31 bdof_vx_vy_8x_padding_left_right v8, v9, v10, v8 bdof_vx_vy_8x_add_line v28, v29, v30, v31, v8 cmp w12, w6 b.ne 10f cbnz x15, 10f // y == block_h && dy == 0, duplicate first line results bdof_vx_vy_8x_add_line v28, v29, v30, v31, v8 add x15, x15, #0x1 b 9b 10: cmp x15, #(BDOF_MIN_BLOCK_SIZE - 1) b.eq 11f cmp x15, #(BDOF_MIN_BLOCK_SIZE) b.ne 12f b 1b 11: // y == BDOF_MIN_BLOCK_SIZE && dy == BDOF_MIN_BLOCK_SIZE - 1 // duplicate the results and break cmp x12, #(BDOF_MIN_BLOCK_SIZE) b.eq 13f bdof_vx_vy_8x_save_line v18, v19, v20, v21, v22 12: add x15, x15, #1 b 8b 13: // y == BDOF_MIN_BLOCK_SIZE && dy == BDOF_MIN_BLOCK_SIZE - 1 // padding bottom then break bdof_vx_vy_8x_add_line v28, v29, v30, v31, v8 b 2b 16: ldp d9, d8, [sp, #0x10] ldp d11, d10, [sp], #0x20 ret endfunc /* * x0: const int16_t *_src0, * x1: const int16_t *_src1, * x2: const int16_t *gradient_h, * x3: const int16_t *gradient_v, * x4: int16_t vx[16], * x5: int16_t vy[16], * w6: int block_h */ function vvc_derive_bdof_vx_vy_16x_neon, export=0 stp d15, d14, [sp, #-0x40]! stp d13, d12, [sp, #0x10] stp d11, d10, [sp, #0x20] stp d9, d8, [sp, #0x30] movrel x12, bdof_vx_vy_16x_tbl ldp q0, q1, [x12] // table mov w13, w6 // y = block_h b 4f 1: // save line4 bdof_vx_vy_16x_save_line v6, v7, v16, v17, v18 2: clz v3.4s, v25.4s mvni v5.4s, #0x1e add v3.4s, v3.4s, v5.4s // -log2() shl v4.4s, v28.4s, #0x2 sshl v3.4s, v4.4s, v3.4s movi v28.4s, #0xf // clip to 15 mvni v29.4s, #0xe // clip to -15 bdof_vx_vy_clip_mask v3, v28, v29, v25, v3 sqxtn v4.4h, v3.4s st1 {v4.d}[0], [x4], #(BDOF_MIN_BLOCK_SIZE * 2) subs x13, x13, #(BDOF_MIN_BLOCK_SIZE) // y -= BDOF_MIN_BLOCK_SIZE clz v4.4s, v24.4s add v4.4s, v4.4s, v5.4s // -log2() shl v5.4s, v27.4s, #0x2 mul v3.4s, v3.4s, v26.4s sshr v3.4s, v3.4s, #0x1 sub v3.4s, v5.4s, v3.4s sshl v3.4s, v3.4s, v4.4s bdof_vx_vy_clip_mask v3, v28, v29, v24, v3 sqxtn v3.4h, v3.4s st1 {v3.d}[0], [x5], #(BDOF_MIN_BLOCK_SIZE * 2) b.eq 16f 4: mov w14, #0x0 // dy, inner loop movi v25.2d, #0 movi v24.2d, #0 movi v26.2d, #0 movi v28.2d, #0 movi v27.2d, #0 b 8f 5: // add line(-1) and line0 from previous results bdof_vx_vy_16x_add_line v19, v20, v21, v22, v23 bdof_vx_vy_16x_add_line v6, v7, v16, v17, v18 add w14, w14, #0x1 8: cmp w13, w6 b.hs 9f // y < block_h && dy == 0, reuse previous results cbz w14, 5b 9: ld1 {v29.8h, v30.8h}, [x0] // src0 sshr v31.8h, v29.8h, #0x4 ld1 {v8.8h, v9.8h}, [x1] // src1 sshr v10.8h, v8.8h, #0x4 ldp q13, q8, [x2], #32 // (gh0 + gh1) >> 1 sshr v29.8h, v30.8h, #0x4 sshr v30.8h, v9.8h, #0x4 ldp q5, q3, [x3], #32 // (gv0 + gv1) >> 1 sub v31.8h, v31.8h, v10.8h // diff, left half sub v4.8h, v29.8h, v30.8h // diff, right half abs v29.8h, v13.8h abs v30.8h, v8.8h abs v9.8h, v5.8h abs v10.8h, v3.8h add x0, x0, #(VVC_MAX_PB_SIZE * 2) add x1, x1, #(VVC_MAX_PB_SIZE * 2) bdof_vx_vy_16x_padding_left_right v29, v30, v11, v12, v14, v29 bdof_vx_vy_16x_padding_left_right v9, v10, v11, v12, v14, v30 bdof_vx_vy_sign v13, v9, v10, v9 bdof_vx_vy_sign v8, v10, v11, v10 bdof_vx_vy_sign v5, v11, v5, v5 bdof_vx_vy_sign v3, v11, v3, v3 mul v11.8h, v5.8h, v13.8h mul v12.8h, v3.8h, v8.8h mul v8.8h, v9.8h, v31.8h mul v9.8h, v10.8h, v4.8h mul v13.8h, v5.8h, v31.8h mul v14.8h, v3.8h, v4.8h bdof_vx_vy_16x_padding_left_right v11, v12, v3, v4, v5, v31 bdof_vx_vy_16x_padding_left_right v8, v9, v3, v4, v5, v8 bdof_vx_vy_16x_padding_left_right v13, v14, v3, v4, v5, v9 bdof_vx_vy_16x_add_line v29, v30, v31, v8, v9 // check whether padding top cmp w13, w6 b.ne 10f cbnz w14, 10f // y == block_h && dy == 0, padding top bdof_vx_vy_16x_add_line v29, v30, v31, v8, v9 add w14, w14, #0x1 b 9b 10: cmp w14, #(BDOF_MIN_BLOCK_SIZE - 1) b.eq 11f cmp w14, #(BDOF_MIN_BLOCK_SIZE) b.ne 12f // save line4 b 1b 11: // y == BDOF_MIN_BLOCK_SIZE && dy == BDOF_MIN_BLOCK_SIZE - 1, padding bottom cmp x13, #(BDOF_MIN_BLOCK_SIZE) b.eq 13f // save line3 bdof_vx_vy_16x_save_line v19, v20, v21, v22, v23 12: add w14, w14, #0x1 // dy++ b 8b 13: // padding bottom bdof_vx_vy_16x_add_line v29, v30, v31, v8, v9 b 2b 16: // restore ldp d9, d8, [sp, #0x30] ldp d11, d10, [sp, #0x20] ldp d13, d12, [sp, #0x10] ldp d15, d14, [sp], #0x40 ret endfunc function ff_vvc_apply_bdof_10_neon, export=1 mov w6, #10 b 0f endfunc function ff_vvc_apply_bdof_12_neon, export=1 mov w6, #12 b 0f endfunc // int16_t gradient_buf_h[2][BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2] // int16_t gradient_buf_v[2][BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2] // int16_t vx[BDOF_BLOCK_SIZE], vy[BDOF_BLOCK_SIZE]; #define APPLY_BDOF_STACK_SIZE ((BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2) * 8 + BDOF_BLOCK_SIZE * 4) #define GRADIENT_H0_OFFSET 2 #define GRADIENT_H1_OFFSET ((BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2) * 2 + 2) #define GRADIENT_V0_OFFSET ((BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2) * 4 + 2) #define GRADIENT_V1_OFFSET ((BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2) * 6 + 2) #define VX_OFFSET ((BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2) * 8) #define VY_OFFSET ((BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2) * 8 + BDOF_BLOCK_SIZE * 2) function ff_vvc_apply_bdof_8_neon, export=1 mov w6, #8 0: stp x19, x20, [sp, #-0x40]! stp x21, x22, [sp, #0x10] stp x23, x24, [sp, #0x20] stp x25, x30, [sp, #0x30] sub sp, sp, #APPLY_BDOF_STACK_SIZE mov w19, w6 // bit_depth mov x20, x0 // dst mov x21, x1 // dst_stride mov x22, x2 // src0 mov x23, x3 // src1 mov w24, w4 // block_w mov w25, w5 // block_h // int16_t *gradient_h[2] = {&gradient_buf_h[0][1], &gradient_buf_h[1][1]}; add x0, sp, #GRADIENT_H0_OFFSET add x1, sp, #GRADIENT_H1_OFFSET add x2, sp, #GRADIENT_V0_OFFSET add x3, sp, #GRADIENT_V1_OFFSET mov x4, x22 mov x5, x23 mov w6, w24 mov w7, w25 bl vvc_bdof_grad_filter_8x_neon cmp w24, #8 mov x0, x22 // src0 mov x1, x23 // src1 add x2, sp, #GRADIENT_H0_OFFSET // gh0 add x3, sp, #GRADIENT_V0_OFFSET // gv0 add x4, sp, #VX_OFFSET // vx add x5, sp, #VY_OFFSET // vy mov w6, w25 // block_h b.gt 16f bl vvc_derive_bdof_vx_vy_8x_neon cmp w19, #10 // check bitdepth mov x0, x20 // dst mov x1, x21 // dst_stride mov x2, x22 // src0 mov x3, x23 // src1 add x4, sp, #GRADIENT_H1_OFFSET // gh1 add x5, sp, #GRADIENT_V1_OFFSET // gv1 add x6, sp, #VX_OFFSET add x7, sp, #VY_OFFSET str w25, [sp] b.eq 1f b.gt 2f // 8bit 0: bl vvc_apply_bdof_block_8x_8_neon b 32f 1: // 10bit bl vvc_apply_bdof_block_8x_10_neon b 32f 2: // 12bit bl vvc_apply_bdof_block_8x_12_neon b 32f 16: bl vvc_derive_bdof_vx_vy_16x_neon cmp w19, #10 // check bitdepth mov x0, x20 // dst mov x1, x21 // dst_stride mov x2, x22 // src0 mov x3, x23 // src1 add x4, sp, #GRADIENT_H1_OFFSET // gh1 add x5, sp, #GRADIENT_V1_OFFSET // gv1 add x6, sp, #VX_OFFSET add x7, sp, #VY_OFFSET str w25, [sp] b.eq 17f b.gt 18f // 8bit bl vvc_apply_bdof_block_16x_8_neon b 32f 17: // 10bit bl vvc_apply_bdof_block_16x_10_neon b 32f 18: // 12bit bl vvc_apply_bdof_block_16x_12_neon 32: add sp, sp, #APPLY_BDOF_STACK_SIZE ldp x25, x30, [sp, #0x30] ldp x23, x24, [sp, #0x20] ldp x21, x22, [sp, #0x10] ldp x19, x20, [sp], #0x40 ret endfunc #undef APPLY_BDOF_STACK_SIZE #undef GRADIENT_H0_OFFSET #undef GRADIENT_H1_OFFSET #undef GRADIENT_V0_OFFSET #undef GRADIENT_V1_OFFSET #undef VX_OFFSET #undef VY_OFFSET #define VVC_MAX_PB_SIZE 128 .macro put_luma_h_x8_vector_filter shift // 8 bytes from hf loaded to v0.8h // 32 bytes from _src loaded to v20.8h & v21.8h where v21.8h is loaded for shift to v1.8h,..,v6.8h,v17.8h // v24.4h & v25.4h are output vectors to store ext v1.16b, v20.16b, v21.16b, #2 ext v2.16b, v20.16b, v21.16b, #4 ext v3.16b, v20.16b, v21.16b, #6 ext v4.16b, v20.16b, v21.16b, #8 ext v5.16b, v20.16b, v21.16b, #10 ext v6.16b, v20.16b, v21.16b, #12 ext v17.16b, v20.16b, v21.16b, #14 smull v24.4s, v20.4h, v0.h[0] smull2 v25.4s, v20.8h, v0.h[0] smlal v24.4s, v1.4h, v0.h[1] smlal2 v25.4s, v1.8h, v0.h[1] smlal v24.4s, v2.4h, v0.h[2] smlal2 v25.4s, v2.8h, v0.h[2] smlal v24.4s, v3.4h, v0.h[3] smlal2 v25.4s, v3.8h, v0.h[3] smlal v24.4s, v4.4h, v0.h[4] smlal2 v25.4s, v4.8h, v0.h[4] smlal v24.4s, v5.4h, v0.h[5] smlal2 v25.4s, v5.8h, v0.h[5] smlal v24.4s, v6.4h, v0.h[6] smlal2 v25.4s, v6.8h, v0.h[6] smlal v24.4s, v17.4h, v0.h[7] smlal2 v25.4s, v17.8h, v0.h[7] sqshrn v24.4h, v24.4s, #(\shift) sqshrn v25.4h, v25.4s, #(\shift) .endm .macro put_luma_h8_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) ld1 {v0.8b}, [x4] sub x1, x1, #6 sxtl v0.8h, v0.8b 1: ld1 {v20.8h, v21.8h}, [x1], x2 put_luma_h_x8_vector_filter \shift subs w3, w3, #1 st1 {v24.4h, v25.4h}, [x0], x9 b.gt 1b ret .endm .macro put_luma_h16_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) ld1 {v0.8b}, [x4] sub x9, x9, #16 sub x1, x1, #6 sxtl v0.8h, v0.8b 1: ld1 {v20.8h, v21.8h, v22.8h}, [x1], x2 put_luma_h_x8_vector_filter \shift mov v20.16b, v21.16b mov v21.16b, v22.16b st1 {v24.4h, v25.4h}, [x0], #16 put_luma_h_x8_vector_filter \shift subs w3, w3, #1 st1 {v24.4h, v25.4h}, [x0], x9 b.gt 1b ret .endm .macro put_luma_h_x16_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) ld1 {v0.8b}, [x4] sub x9, x9, w6, uxtw #1 sub x2, x2, w6, uxtw #1 sxtl v0.8h, v0.8b sub x1, x1, #6 sub x2, x2, #16 1: ld1 {v20.8h}, [x1], #16 mov w8, w6 2: ld1 {v21.8h, v22.8h}, [x1], #32 put_luma_h_x8_vector_filter \shift mov v20.16b, v21.16b mov v21.16b, v22.16b st1 {v24.4h, v25.4h}, [x0], #16 put_luma_h_x8_vector_filter \shift mov v20.16b, v21.16b subs w8, w8, #16 st1 {v24.4h, v25.4h}, [x0], #16 b.gt 2b subs w3, w3, #1 add x0, x0, x9 add x1, x1, x2 b.gt 1b ret .endm function ff_vvc_put_luma_h8_10_neon, export=1 put_luma_h8_xx_neon 2 endfunc function ff_vvc_put_luma_h8_12_neon, export=1 put_luma_h8_xx_neon 4 endfunc function ff_vvc_put_luma_h16_10_neon, export=1 put_luma_h16_xx_neon 2 endfunc function ff_vvc_put_luma_h16_12_neon, export=1 put_luma_h16_xx_neon 4 endfunc function ff_vvc_put_luma_h_x16_10_neon, export=1 put_luma_h_x16_xx_neon 2 endfunc function ff_vvc_put_luma_h_x16_12_neon, export=1 put_luma_h_x16_xx_neon 4 endfunc .macro put_luma_v4_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, x2, lsl #1 ld1 {v0.8b}, [x5] sub x1, x1, x2 sxtl v0.8h, v0.8b ld1 {v20.4h}, [x1], x2 ld1 {v21.4h}, [x1], x2 ld1 {v22.4h}, [x1], x2 ld1 {v23.4h}, [x1], x2 ld1 {v24.4h}, [x1], x2 ld1 {v25.4h}, [x1], x2 ld1 {v26.4h}, [x1], x2 1: ld1 {v27.4h}, [x1], x2 smull v1.4s, v20.4h, v0.h[0] smull v2.4s, v21.4h, v0.h[1] smlal v1.4s, v22.4h, v0.h[2] smlal v2.4s, v23.4h, v0.h[3] smlal v1.4s, v24.4h, v0.h[4] smlal v2.4s, v25.4h, v0.h[5] smlal v1.4s, v26.4h, v0.h[6] smlal v2.4s, v27.4h, v0.h[7] ld1 {v28.4h}, [x1], x2 smull v3.4s, v21.4h, v0.h[0] smull v4.4s, v22.4h, v0.h[1] smlal v3.4s, v23.4h, v0.h[2] smlal v4.4s, v24.4h, v0.h[3] smlal v3.4s, v25.4h, v0.h[4] smlal v4.4s, v26.4h, v0.h[5] smlal v3.4s, v27.4h, v0.h[6] smlal v4.4s, v28.4h, v0.h[7] add v1.4s, v1.4s, v2.4s add v3.4s, v3.4s, v4.4s sqshrn v1.4h, v1.4s, #(\shift) sqshrn v3.4h, v3.4s, #(\shift) st1 {v1.4h}, [x0], x9 ld1 {v29.4h}, [x1], x2 st1 {v3.4h}, [x0], x9 smull v1.4s, v22.4h, v0.h[0] smull v2.4s, v23.4h, v0.h[1] smlal v1.4s, v24.4h, v0.h[2] smlal v2.4s, v25.4h, v0.h[3] smlal v1.4s, v26.4h, v0.h[4] smlal v2.4s, v27.4h, v0.h[5] smlal v1.4s, v28.4h, v0.h[6] smlal v2.4s, v29.4h, v0.h[7] ld1 {v30.4h}, [x1], x2 smull v3.4s, v23.4h, v0.h[0] smull v4.4s, v24.4h, v0.h[1] smlal v3.4s, v25.4h, v0.h[2] smlal v4.4s, v26.4h, v0.h[3] smlal v3.4s, v27.4h, v0.h[4] smlal v4.4s, v28.4h, v0.h[5] smlal v3.4s, v29.4h, v0.h[6] smlal v4.4s, v30.4h, v0.h[7] add v1.4s, v1.4s, v2.4s add v3.4s, v3.4s, v4.4s sqshrn v1.4h, v1.4s, #(\shift) sqshrn v3.4h, v3.4s, #(\shift) st1 {v1.4h}, [x0], x9 mov v20.16b, v24.16b mov v21.16b, v25.16b mov v22.16b, v26.16b mov v23.16b, v27.16b mov v24.16b, v28.16b mov v25.16b, v29.16b mov v26.16b, v30.16b subs w3, w3, #4 st1 {v3.4h}, [x0], x9 b.gt 1b ret .endm function ff_vvc_put_luma_v4_10_neon, export=1 put_luma_v4_xx_neon 2 endfunc function ff_vvc_put_luma_v4_12_neon, export=1 put_luma_v4_xx_neon 4 endfunc .macro put_luma_v8_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, x2, lsl #1 ld1 {v0.8b}, [x5] sub x1, x1, x2 sxtl v0.8h, v0.8b ld1 {v20.8h}, [x1], x2 ld1 {v21.8h}, [x1], x2 ld1 {v22.8h}, [x1], x2 ld1 {v23.8h}, [x1], x2 ld1 {v24.8h}, [x1], x2 ld1 {v25.8h}, [x1], x2 ld1 {v26.8h}, [x1], x2 1: ld1 {v27.8h}, [x1], x2 smull v1.4s, v20.4h, v0.h[0] smull2 v2.4s, v20.8h, v0.h[0] smlal v1.4s, v21.4h, v0.h[1] smlal2 v2.4s, v21.8h, v0.h[1] smlal v1.4s, v22.4h, v0.h[2] smlal2 v2.4s, v22.8h, v0.h[2] smlal v1.4s, v23.4h, v0.h[3] smlal2 v2.4s, v23.8h, v0.h[3] smlal v1.4s, v24.4h, v0.h[4] smlal2 v2.4s, v24.8h, v0.h[4] smlal v1.4s, v25.4h, v0.h[5] smlal2 v2.4s, v25.8h, v0.h[5] smlal v1.4s, v26.4h, v0.h[6] smlal2 v2.4s, v26.8h, v0.h[6] smlal v1.4s, v27.4h, v0.h[7] smlal2 v2.4s, v27.8h, v0.h[7] sqshrn v1.4h, v1.4s, #(\shift) sqshrn v2.4h, v2.4s, #(\shift) ld1 {v28.8h}, [x1], x2 st1 {v1.4h-v2.4h}, [x0], x9 smull v3.4s, v21.4h, v0.h[0] smull2 v4.4s, v21.8h, v0.h[0] smlal v3.4s, v22.4h, v0.h[1] smlal2 v4.4s, v22.8h, v0.h[1] smlal v3.4s, v23.4h, v0.h[2] smlal2 v4.4s, v23.8h, v0.h[2] smlal v3.4s, v24.4h, v0.h[3] smlal2 v4.4s, v24.8h, v0.h[3] smlal v3.4s, v25.4h, v0.h[4] smlal2 v4.4s, v25.8h, v0.h[4] smlal v3.4s, v26.4h, v0.h[5] smlal2 v4.4s, v26.8h, v0.h[5] smlal v3.4s, v27.4h, v0.h[6] smlal2 v4.4s, v27.8h, v0.h[6] smlal v3.4s, v28.4h, v0.h[7] smlal2 v4.4s, v28.8h, v0.h[7] sqshrn v3.4h, v3.4s, #(\shift) sqshrn v4.4h, v4.4s, #(\shift) ld1 {v29.8h}, [x1], x2 st1 {v3.4h-v4.4h}, [x0], x9 smull v1.4s, v22.4h, v0.h[0] smull2 v2.4s, v22.8h, v0.h[0] smlal v1.4s, v23.4h, v0.h[1] smlal2 v2.4s, v23.8h, v0.h[1] smlal v1.4s, v24.4h, v0.h[2] smlal2 v2.4s, v24.8h, v0.h[2] smlal v1.4s, v25.4h, v0.h[3] smlal2 v2.4s, v25.8h, v0.h[3] smlal v1.4s, v26.4h, v0.h[4] smlal2 v2.4s, v26.8h, v0.h[4] smlal v1.4s, v27.4h, v0.h[5] smlal2 v2.4s, v27.8h, v0.h[5] smlal v1.4s, v28.4h, v0.h[6] smlal2 v2.4s, v28.8h, v0.h[6] smlal v1.4s, v29.4h, v0.h[7] smlal2 v2.4s, v29.8h, v0.h[7] sqshrn v1.4h, v1.4s, #(\shift) sqshrn v2.4h, v2.4s, #(\shift) ld1 {v30.8h}, [x1], x2 st1 {v1.4h-v2.4h}, [x0], x9 smull v3.4s, v23.4h, v0.h[0] smull2 v4.4s, v23.8h, v0.h[0] smlal v3.4s, v24.4h, v0.h[1] smlal2 v4.4s, v24.8h, v0.h[1] smlal v3.4s, v25.4h, v0.h[2] smlal2 v4.4s, v25.8h, v0.h[2] smlal v3.4s, v26.4h, v0.h[3] smlal2 v4.4s, v26.8h, v0.h[3] smlal v3.4s, v27.4h, v0.h[4] smlal2 v4.4s, v27.8h, v0.h[4] smlal v3.4s, v28.4h, v0.h[5] smlal2 v4.4s, v28.8h, v0.h[5] smlal v3.4s, v29.4h, v0.h[6] smlal2 v4.4s, v29.8h, v0.h[6] smlal v3.4s, v30.4h, v0.h[7] smlal2 v4.4s, v30.8h, v0.h[7] sqshrn v3.4h, v3.4s, #(\shift) sqshrn v4.4h, v4.4s, #(\shift) mov v20.16b, v24.16b mov v21.16b, v25.16b mov v22.16b, v26.16b mov v23.16b, v27.16b mov v24.16b, v28.16b mov v25.16b, v29.16b mov v26.16b, v30.16b subs w3, w3, #4 st1 {v3.4h-v4.4h}, [x0], x9 b.gt 1b ret .endm function ff_vvc_put_luma_v8_10_neon, export=1 put_luma_v8_xx_neon 2 endfunc function ff_vvc_put_luma_v8_12_neon, export=1 put_luma_v8_xx_neon 4 endfunc .macro put_luma_v_x16_vector_filter shift smull v2.4s, v16.4h, v1.h[0] smull2 v3.4s, v16.8h, v1.h[0] smlal v2.4s, v18.4h, v1.h[1] smlal2 v3.4s, v18.8h, v1.h[1] smlal v2.4s, v20.4h, v1.h[2] smlal2 v3.4s, v20.8h, v1.h[2] smlal v2.4s, v22.4h, v1.h[3] smlal2 v3.4s, v22.8h, v1.h[3] smlal v2.4s, v24.4h, v1.h[4] smlal2 v3.4s, v24.8h, v1.h[4] smlal v2.4s, v26.4h, v1.h[5] smlal2 v3.4s, v26.8h, v1.h[5] smlal v2.4s, v28.4h, v1.h[6] smlal2 v3.4s, v28.8h, v1.h[6] smlal v2.4s, v30.4h, v1.h[7] smlal2 v3.4s, v30.8h, v1.h[7] smull v4.4s, v17.4h, v1.h[0] smull2 v5.4s, v17.8h, v1.h[0] smlal v4.4s, v19.4h, v1.h[1] smlal2 v5.4s, v19.8h, v1.h[1] smlal v4.4s, v21.4h, v1.h[2] smlal2 v5.4s, v21.8h, v1.h[2] smlal v4.4s, v23.4h, v1.h[3] smlal2 v5.4s, v23.8h, v1.h[3] smlal v4.4s, v25.4h, v1.h[4] smlal2 v5.4s, v25.8h, v1.h[4] smlal v4.4s, v27.4h, v1.h[5] smlal2 v5.4s, v27.8h, v1.h[5] smlal v4.4s, v29.4h, v1.h[6] smlal2 v5.4s, v29.8h, v1.h[6] smlal v4.4s, v31.4h, v1.h[7] smlal2 v5.4s, v31.8h, v1.h[7] sqshrn v6.4h, v2.4s, #(\shift) sqshrn v7.4h, v4.4s, #(\shift) sqshrn2 v6.8h, v3.4s, #(\shift) sqshrn2 v7.8h, v5.4s, #(\shift) .endm .macro put_luma_v16_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, x2, lsl #1 ld1 {v0.8b}, [x5] sub x1, x1, x2 sxtl v0.8h, v0.8b ld1 {v16.8h-v17.8h}, [x1], x2 ld1 {v18.8h-v19.8h}, [x1], x2 ld1 {v20.8h-v21.8h}, [x1], x2 ld1 {v22.8h-v23.8h}, [x1], x2 ld1 {v24.8h-v25.8h}, [x1], x2 ld1 {v26.8h-v27.8h}, [x1], x2 ld1 {v28.8h-v29.8h}, [x1], x2 1: mov v1.16b, v0.16b ld1 {v30.8h-v31.8h}, [x1], x2 put_luma_v_x16_vector_filter \shift ld1 {v16.8h-v17.8h}, [x1], x2 ext v1.16b, v0.16b, v0.16b, #14 st1 {v6.8h-v7.8h}, [x0], x9 put_luma_v_x16_vector_filter \shift ld1 {v18.8h-v19.8h}, [x1], x2 ext v1.16b, v0.16b, v0.16b, #12 st1 {v6.8h-v7.8h}, [x0], x9 put_luma_v_x16_vector_filter \shift ld1 {v20.8h-v21.8h}, [x1], x2 ext v1.16b, v0.16b, v0.16b, #10 st1 {v6.8h-v7.8h}, [x0], x9 put_luma_v_x16_vector_filter \shift subs w3, w3, #4 st1 {v6.8h-v7.8h}, [x0], x9 mov v2.16b, v16.16b mov v3.16b, v17.16b mov v16.16b, v24.16b mov v17.16b, v25.16b mov v24.16b, v2.16b mov v25.16b, v3.16b mov v2.16b, v18.16b mov v3.16b, v19.16b mov v18.16b, v26.16b mov v19.16b, v27.16b mov v26.16b, v2.16b mov v27.16b, v3.16b mov v2.16b, v20.16b mov v3.16b, v21.16b mov v20.16b, v28.16b mov v21.16b, v29.16b mov v28.16b, v2.16b mov v29.16b, v3.16b mov v22.16b, v30.16b mov v23.16b, v31.16b b.gt 1b ret .endm function ff_vvc_put_luma_v16_10_neon, export=1 put_luma_v16_xx_neon 2 endfunc function ff_vvc_put_luma_v16_12_neon, export=1 put_luma_v16_xx_neon 4 endfunc .macro put_luma_v_x16_xx_neon shift mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, x2, lsl #1 ld1 {v0.8b}, [x5] sub x1, x1, x2 sxtl v0.8h, v0.8b 1: mov w8, #0 2: add x11, x1, x8, lsl #1 add x10, x0, x8, lsl #1 ld1 {v16.8h-v17.8h}, [x11], x2 add x8, x8, #16 ld1 {v18.8h-v19.8h}, [x11], x2 cmp w8, w6 ld1 {v20.8h-v21.8h}, [x11], x2 mov v1.16b, v0.16b ld1 {v22.8h-v23.8h}, [x11], x2 ld1 {v24.8h-v25.8h}, [x11], x2 ld1 {v26.8h-v27.8h}, [x11], x2 ld1 {v28.8h-v29.8h}, [x11], x2 ld1 {v30.8h-v31.8h}, [x11], x2 put_luma_v_x16_vector_filter \shift ld1 {v16.8h-v17.8h}, [x11], x2 ext v1.16b, v0.16b, v0.16b, #14 st1 {v6.8h-v7.8h}, [x10], x9 put_luma_v_x16_vector_filter \shift st1 {v6.8h-v7.8h}, [x10], x9 ext v1.16b, v0.16b, v0.16b, #12 ld1 {v18.8h-v19.8h}, [x11], x2 put_luma_v_x16_vector_filter \shift ld1 {v20.8h-v21.8h}, [x11], x2 ext v1.16b, v0.16b, v0.16b, #10 st1 {v6.8h-v7.8h}, [x10], x9 put_luma_v_x16_vector_filter \shift st1 {v6.8h-v7.8h}, [x10], x9 b.lt 2b add x0, x0, x9, lsl #2 subs w3, w3, #4 add x1, x1, x2, lsl #2 b.gt 1b ret .endm function ff_vvc_put_luma_v_x16_10_neon, export=1 put_luma_v_x16_xx_neon 2 endfunc function ff_vvc_put_luma_v_x16_12_neon, export=1 put_luma_v_x16_xx_neon 4 endfunc .macro put_luma_hv_x8_horizontal_filter shift, dst, src0, src1 ext v2.16b, \src0\().16b, \src1\().16b, #2 ext v3.16b, \src0\().16b, \src1\().16b, #4 ext v4.16b, \src0\().16b, \src1\().16b, #6 ext v5.16b, \src0\().16b, \src1\().16b, #8 smull v6.4s, \src0\().4h, v0.h[0] smull2 v7.4s, \src0\().8h, v0.h[0] smlal v6.4s, v2.4h, v0.h[1] smlal2 v7.4s, v2.8h, v0.h[1] smlal v6.4s, v3.4h, v0.h[2] smlal2 v7.4s, v3.8h, v0.h[2] smlal v6.4s, v4.4h, v0.h[3] smlal2 v7.4s, v4.8h, v0.h[3] smlal v6.4s, v5.4h, v0.h[4] smlal2 v7.4s, v5.8h, v0.h[4] ext v2.16b, \src0\().16b, \src1\().16b, #10 ext v3.16b, \src0\().16b, \src1\().16b, #12 ext v4.16b, \src0\().16b, \src1\().16b, #14 smlal v6.4s, v2.4h, v0.h[5] smlal2 v7.4s, v2.8h, v0.h[5] smlal v6.4s, v3.4h, v0.h[6] smlal2 v7.4s, v3.8h, v0.h[6] smlal v6.4s, v4.4h, v0.h[7] smlal2 v7.4s, v4.8h, v0.h[7] sqshrn \dst\().4h, v6.4s, #(\shift) sqshrn2 \dst\().8h, v7.4s, #(\shift) .endm .macro put_luma_hv_x8_vertical_filter dst0, dst1, src0, src1, src2, src3, src4, src5, src6, src7 smull \dst0\().4s, \src0\().4h, v1.h[0] smull2 \dst1\().4s, \src0\().8h, v1.h[0] smlal \dst0\().4s, \src1\().4h, v1.h[1] smlal2 \dst1\().4s, \src1\().8h, v1.h[1] smlal \dst0\().4s, \src2\().4h, v1.h[2] smlal2 \dst1\().4s, \src2\().8h, v1.h[2] smlal \dst0\().4s, \src3\().4h, v1.h[3] smlal2 \dst1\().4s, \src3\().8h, v1.h[3] smlal \dst0\().4s, \src4\().4h, v1.h[4] smlal2 \dst1\().4s, \src4\().8h, v1.h[4] smlal \dst0\().4s, \src5\().4h, v1.h[5] smlal2 \dst1\().4s, \src5\().8h, v1.h[5] smlal \dst0\().4s, \src6\().4h, v1.h[6] smlal2 \dst1\().4s, \src6\().8h, v1.h[6] smlal \dst0\().4s, \src7\().4h, v1.h[7] smlal2 \dst1\().4s, \src7\().8h, v1.h[7] sqshrn \dst0\().4h, \dst0\().4s, #6 sqshrn \dst1\().4h, \dst1\().4s, #6 .endm .macro put_luma_hv8_xx_neon shift // dst .req x0 // _src .req x1 // _src_stride .req x2 // height .req x3 // hf .req x4 // vf .req x5 // width .req x6 mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, #6 ld1 {v0.8b}, [x4] sub x1, x1, x2, lsl #1 sxtl v0.8h, v0.8b ld1 {v1.8b}, [x5] sub x1, x1, x2 sxtl v1.8h, v1.8b ld1 {v16.8h, v17.8h}, [x1], x2 ld1 {v18.8h, v19.8h}, [x1], x2 ld1 {v20.8h, v21.8h}, [x1], x2 ld1 {v22.8h, v23.8h}, [x1], x2 ld1 {v24.8h, v25.8h}, [x1], x2 ld1 {v26.8h, v27.8h}, [x1], x2 ld1 {v28.8h, v29.8h}, [x1], x2 put_luma_hv_x8_horizontal_filter \shift, v16, v16, v17 put_luma_hv_x8_horizontal_filter \shift, v18, v18, v19 put_luma_hv_x8_horizontal_filter \shift, v20, v20, v21 put_luma_hv_x8_horizontal_filter \shift, v22, v22, v23 put_luma_hv_x8_horizontal_filter \shift, v24, v24, v25 put_luma_hv_x8_horizontal_filter \shift, v26, v26, v27 put_luma_hv_x8_horizontal_filter \shift, v28, v28, v29 1: ld1 {v30.8h, v31.8h}, [x1], x2 put_luma_hv_x8_horizontal_filter \shift, v30, v30, v31 put_luma_hv_x8_vertical_filter v2, v3, v16, v18, v20, v22, v24, v26, v28, v30 ld1 {v16.8h, v17.8h}, [x1], x2 st1 {v2.4h-v3.4h}, [x0], x9 put_luma_hv_x8_horizontal_filter \shift, v16, v16, v17 put_luma_hv_x8_vertical_filter v2, v3, v18, v20, v22, v24, v26, v28, v30, v16 ld1 {v18.8h, v19.8h}, [x1], x2 st1 {v2.4h-v3.4h}, [x0], x9 put_luma_hv_x8_horizontal_filter \shift, v18, v18, v19 put_luma_hv_x8_vertical_filter v2, v3, v20, v22, v24, v26, v28, v30, v16, v18 ld1 {v20.8h, v21.8h}, [x1], x2 st1 {v2.4h-v3.4h}, [x0], x9 put_luma_hv_x8_horizontal_filter \shift, v20, v20, v21 put_luma_hv_x8_vertical_filter v2, v3, v22, v24, v26, v28, v30, v16, v18, v20 st1 {v2.4h-v3.4h}, [x0], x9 mov v17.16b, v16.16b mov v16.16b, v24.16b mov v24.16b, v17.16b mov v19.16b, v18.16b mov v18.16b, v26.16b mov v26.16b, v19.16b mov v21.16b, v20.16b mov v20.16b, v28.16b mov v28.16b, v21.16b subs w3, w3, #4 mov v22.16b, v30.16b b.gt 1b ret .endm function ff_vvc_put_luma_hv8_10_neon, export=1 put_luma_hv8_xx_neon 2 endfunc function ff_vvc_put_luma_hv8_12_neon, export=1 put_luma_hv8_xx_neon 4 endfunc .macro put_luma_hv16_xx_neon shift // dst .req x0 // _src .req x1 // _src_stride .req x2 // height .req x3 // hf .req x4 // vf .req x5 // width .req x6 stp d8, d9, [sp, #-64]! stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, #6 ld1 {v0.8b}, [x4] sub x1, x1, x2, lsl #1 sxtl v0.8h, v0.8b ld1 {v1.8b}, [x5] sub x1, x1, x2 sxtl v1.8h, v1.8b ld1 {v8.8h, v9.8h, v10.8h}, [x1], x2 ld1 {v11.8h, v12.8h, v13.8h}, [x1], x2 ld1 {v14.8h, v15.8h, v16.8h}, [x1], x2 ld1 {v17.8h, v18.8h, v19.8h}, [x1], x2 ld1 {v20.8h, v21.8h, v22.8h}, [x1], x2 ld1 {v23.8h, v24.8h, v25.8h}, [x1], x2 ld1 {v26.8h, v27.8h, v28.8h}, [x1], x2 put_luma_hv_x8_horizontal_filter \shift, v8, v8, v9 put_luma_hv_x8_horizontal_filter \shift, v9, v9, v10 put_luma_hv_x8_horizontal_filter \shift, v11, v11, v12 put_luma_hv_x8_horizontal_filter \shift, v12, v12, v13 put_luma_hv_x8_horizontal_filter \shift, v14, v14, v15 put_luma_hv_x8_horizontal_filter \shift, v15, v15, v16 put_luma_hv_x8_horizontal_filter \shift, v17, v17, v18 put_luma_hv_x8_horizontal_filter \shift, v18, v18, v19 put_luma_hv_x8_horizontal_filter \shift, v20, v20, v21 put_luma_hv_x8_horizontal_filter \shift, v21, v21, v22 put_luma_hv_x8_horizontal_filter \shift, v23, v23, v24 put_luma_hv_x8_horizontal_filter \shift, v24, v24, v25 put_luma_hv_x8_horizontal_filter \shift, v26, v26, v27 put_luma_hv_x8_horizontal_filter \shift, v27, v27, v28 1: ld1 {v29.8h, v30.8h, v31.8h}, [x1], x2 put_luma_hv_x8_horizontal_filter \shift, v29, v29, v30 put_luma_hv_x8_horizontal_filter \shift, v30, v30, v31 put_luma_hv_x8_vertical_filter v2, v3, v8, v11, v14, v17, v20, v23, v26, v29 put_luma_hv_x8_vertical_filter v4, v5, v9, v12, v15, v18, v21, v24, v27, v30 ld1 {v8.8h, v9.8h, v10.8h}, [x1], x2 st1 {v2.4h-v5.4h}, [x0], x9 put_luma_hv_x8_horizontal_filter \shift, v8, v8, v9 put_luma_hv_x8_horizontal_filter \shift, v9, v9, v10 put_luma_hv_x8_vertical_filter v2, v3, v11, v14, v17, v20, v23, v26, v29, v8 put_luma_hv_x8_vertical_filter v4, v5, v12, v15, v18, v21, v24, v27, v30, v9 ld1 {v11.8h, v12.8h, v13.8h}, [x1], x2 st1 {v2.4h-v5.4h}, [x0], x9 put_luma_hv_x8_horizontal_filter \shift, v11, v11, v12 put_luma_hv_x8_horizontal_filter \shift, v12, v12, v13 put_luma_hv_x8_vertical_filter v2, v3, v14, v17, v20, v23, v26, v29, v8, v11 put_luma_hv_x8_vertical_filter v4, v5, v15, v18, v21, v24, v27, v30, v9, v12 ld1 {v14.8h, v15.8h, v16.8h}, [x1], x2 st1 {v2.4h-v5.4h}, [x0], x9 put_luma_hv_x8_horizontal_filter \shift, v14, v14, v15 put_luma_hv_x8_horizontal_filter \shift, v15, v15, v16 put_luma_hv_x8_vertical_filter v2, v3, v17, v20, v23, v26, v29, v8, v11, v14 put_luma_hv_x8_vertical_filter v4, v5, v18, v21, v24, v27, v30, v9, v12, v15 st1 {v2.4h-v5.4h}, [x0], x9 mov v10.16b, v8.16b mov v8.16b, v20.16b mov v20.16b, v10.16b mov v10.16b, v9.16b mov v9.16b, v21.16b mov v21.16b, v10.16b mov v13.16b, v11.16b mov v11.16b, v23.16b mov v23.16b, v13.16b mov v13.16b, v12.16b mov v12.16b, v24.16b mov v24.16b, v13.16b mov v16.16b, v14.16b mov v14.16b, v26.16b mov v26.16b, v16.16b mov v16.16b, v15.16b mov v15.16b, v27.16b mov v27.16b, v16.16b subs w3, w3, #4 mov v17.16b, v29.16b mov v18.16b, v30.16b b.gt 1b ldp d14, d15, [sp, #48] ldp d12, d13, [sp, #32] ldp d10, d11, [sp, #16] ldp d8, d9, [sp], #64 ret .endm function ff_vvc_put_luma_hv16_10_neon, export=1 put_luma_hv16_xx_neon 2 endfunc function ff_vvc_put_luma_hv16_12_neon, export=1 put_luma_hv16_xx_neon 4 endfunc .macro put_luma_hv_x16_xx_neon shift // dst .req x0 // _src .req x1 // _src_stride .req x2 // height .req x3 // hf .req x4 // vf .req x5 // width .req x6 uxtw x6, w6 stp d8, d9, [sp, #-64]! stp d10, d11, [sp, #16] stp d12, d13, [sp, #32] stp d14, d15, [sp, #48] mov x9, #(VVC_MAX_PB_SIZE * 2) sub x1, x1, #6 ld1 {v0.8b}, [x4] sub x1, x1, x2, lsl #1 sxtl v0.8h, v0.8b ld1 {v1.8b}, [x5] sub x1, x1, x2 sxtl v1.8h, v1.8b 1: mov w13, w3 mov x11, x1 mov x10, x0 ld1 {v8.8h, v9.8h, v10.8h}, [x11], x2 ld1 {v11.8h, v12.8h, v13.8h}, [x11], x2 ld1 {v14.8h, v15.8h, v16.8h}, [x11], x2 ld1 {v17.8h, v18.8h, v19.8h}, [x11], x2 ld1 {v20.8h, v21.8h, v22.8h}, [x11], x2 ld1 {v23.8h, v24.8h, v25.8h}, [x11], x2 ld1 {v26.8h, v27.8h, v28.8h}, [x11], x2 put_luma_hv_x8_horizontal_filter \shift, v8, v8, v9 put_luma_hv_x8_horizontal_filter \shift, v9, v9, v10 put_luma_hv_x8_horizontal_filter \shift, v11, v11, v12 put_luma_hv_x8_horizontal_filter \shift, v12, v12, v13 put_luma_hv_x8_horizontal_filter \shift, v14, v14, v15 put_luma_hv_x8_horizontal_filter \shift, v15, v15, v16 put_luma_hv_x8_horizontal_filter \shift, v17, v17, v18 put_luma_hv_x8_horizontal_filter \shift, v18, v18, v19 put_luma_hv_x8_horizontal_filter \shift, v20, v20, v21 put_luma_hv_x8_horizontal_filter \shift, v21, v21, v22 put_luma_hv_x8_horizontal_filter \shift, v23, v23, v24 put_luma_hv_x8_horizontal_filter \shift, v24, v24, v25 put_luma_hv_x8_horizontal_filter \shift, v26, v26, v27 put_luma_hv_x8_horizontal_filter \shift, v27, v27, v28 2: ld1 {v29.8h, v30.8h, v31.8h}, [x11], x2 put_luma_hv_x8_horizontal_filter \shift, v29, v29, v30 put_luma_hv_x8_horizontal_filter \shift, v30, v30, v31 put_luma_hv_x8_vertical_filter v2, v3, v8, v11, v14, v17, v20, v23, v26, v29 put_luma_hv_x8_vertical_filter v4, v5, v9, v12, v15, v18, v21, v24, v27, v30 ld1 {v8.8h, v9.8h, v10.8h}, [x11], x2 st1 {v2.4h-v5.4h}, [x10], x9 put_luma_hv_x8_horizontal_filter \shift, v8, v8, v9 put_luma_hv_x8_horizontal_filter \shift, v9, v9, v10 put_luma_hv_x8_vertical_filter v2, v3, v11, v14, v17, v20, v23, v26, v29, v8 put_luma_hv_x8_vertical_filter v4, v5, v12, v15, v18, v21, v24, v27, v30, v9 ld1 {v11.8h, v12.8h, v13.8h}, [x11], x2 st1 {v2.4h-v5.4h}, [x10], x9 put_luma_hv_x8_horizontal_filter \shift, v11, v11, v12 put_luma_hv_x8_horizontal_filter \shift, v12, v12, v13 put_luma_hv_x8_vertical_filter v2, v3, v14, v17, v20, v23, v26, v29, v8, v11 put_luma_hv_x8_vertical_filter v4, v5, v15, v18, v21, v24, v27, v30, v9, v12 ld1 {v14.8h, v15.8h, v16.8h}, [x11], x2 st1 {v2.4h-v5.4h}, [x10], x9 put_luma_hv_x8_horizontal_filter \shift, v14, v14, v15 put_luma_hv_x8_horizontal_filter \shift, v15, v15, v16 put_luma_hv_x8_vertical_filter v2, v3, v17, v20, v23, v26, v29, v8, v11, v14 put_luma_hv_x8_vertical_filter v4, v5, v18, v21, v24, v27, v30, v9, v12, v15 st1 {v2.4h-v5.4h}, [x10], x9 mov v10.16b, v8.16b mov v8.16b, v20.16b mov v20.16b, v10.16b mov v10.16b, v9.16b mov v9.16b, v21.16b mov v21.16b, v10.16b mov v13.16b, v11.16b mov v11.16b, v23.16b mov v23.16b, v13.16b mov v13.16b, v12.16b mov v12.16b, v24.16b mov v24.16b, v13.16b mov v16.16b, v14.16b mov v14.16b, v26.16b mov v26.16b, v16.16b mov v16.16b, v15.16b mov v15.16b, v27.16b mov v27.16b, v16.16b subs w13, w13, #4 mov v17.16b, v29.16b mov v18.16b, v30.16b b.gt 2b add x0, x0, #32 add x1, x1, #32 subs w6, w6, #16 b.gt 1b ldp d14, d15, [sp, #48] ldp d12, d13, [sp, #32] ldp d10, d11, [sp, #16] ldp d8, d9, [sp], #64 ret .endm function ff_vvc_put_luma_hv_x16_10_neon, export=1 put_luma_hv_x16_xx_neon 2 endfunc function ff_vvc_put_luma_hv_x16_12_neon, export=1 put_luma_hv_x16_xx_neon 4 endfunc