Add Armv8.4 Neon DotProd path for av1_dist_wtd_convolve_y Implement Armv8.4 DotProd path for av1_dist_wtd_convolve_y and add the associated unit tests. Change-Id: If404e34e05ace598df9b63902059218bcefdceaf
diff --git a/av1/common/arm/compound_convolve_neon.h b/av1/common/arm/compound_convolve_neon.h index e9837f0..66d0d58 100644 --- a/av1/common/arm/compound_convolve_neon.h +++ b/av1/common/arm/compound_convolve_neon.h
@@ -124,6 +124,46 @@ *d23_u8 = vqrshrun_n_s16(dst_23, FILTER_BITS - ROUND0_BITS); } +static inline void compute_dist_wtd_avg_8x2( + uint16x8_t dd0, uint16x8_t dd1, uint16x8_t d0, uint16x8_t d1, + const uint16_t fwd_offset, const uint16_t bck_offset, + const int16x8_t round_offset, uint8x8_t *d0_u8, uint8x8_t *d1_u8) { + uint32x4_t blend0_lo = vmull_n_u16(vget_low_u16(dd0), fwd_offset); + blend0_lo = vmlal_n_u16(blend0_lo, vget_low_u16(d0), bck_offset); + uint32x4_t blend0_hi = vmull_n_u16(vget_high_u16(dd0), fwd_offset); + blend0_hi = vmlal_n_u16(blend0_hi, vget_high_u16(d0), bck_offset); + + uint32x4_t blend1_lo = vmull_n_u16(vget_low_u16(dd1), fwd_offset); + blend1_lo = vmlal_n_u16(blend1_lo, vget_low_u16(d1), bck_offset); + uint32x4_t blend1_hi = vmull_n_u16(vget_high_u16(dd1), fwd_offset); + blend1_hi = vmlal_n_u16(blend1_hi, vget_high_u16(d1), bck_offset); + + uint16x8_t avg0 = vcombine_u16(vshrn_n_u32(blend0_lo, DIST_PRECISION_BITS), + vshrn_n_u32(blend0_hi, DIST_PRECISION_BITS)); + uint16x8_t avg1 = vcombine_u16(vshrn_n_u32(blend1_lo, DIST_PRECISION_BITS), + vshrn_n_u32(blend1_hi, DIST_PRECISION_BITS)); + + int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset); + int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset); + + *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS); + *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS); +} + +static inline void compute_basic_avg_8x2(uint16x8_t dd01, uint16x8_t dd23, + uint16x8_t d01, uint16x8_t d23, + const int16x8_t round_offset, + uint8x8_t *d0_u8, uint8x8_t *d1_u8) { + uint16x8_t avg0 = vhaddq_u16(dd01, d01); + uint16x8_t avg1 = vhaddq_u16(dd23, d23); + + int16x8_t dst0 = vsubq_s16(vreinterpretq_s16_u16(avg0), round_offset); + int16x8_t dst1 = vsubq_s16(vreinterpretq_s16_u16(avg1), round_offset); + + *d0_u8 = vqrshrun_n_s16(dst0, FILTER_BITS - ROUND0_BITS); + *d1_u8 = vqrshrun_n_s16(dst1, FILTER_BITS - ROUND0_BITS); +} + static inline void compute_dist_wtd_avg_8x4( uint16x8_t dd0, uint16x8_t dd1, uint16x8_t dd2, uint16x8_t dd3, uint16x8_t d0, uint16x8_t d1, uint16x8_t d2, uint16x8_t d3,
diff --git a/av1/common/arm/compound_convolve_neon_dotprod.c b/av1/common/arm/compound_convolve_neon_dotprod.c index dcf8da1..76a1d0b 100644 --- a/av1/common/arm/compound_convolve_neon_dotprod.c +++ b/av1/common/arm/compound_convolve_neon_dotprod.c
@@ -13,6 +13,7 @@ #include <assert.h> #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "av1/common/arm/compound_convolve_neon.h" #include "config/aom_config.h" #include "config/av1_rtcd.h" @@ -23,6 +24,17 @@ 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 }; +DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { + // clang-format off + // Shift left and insert new last column in transposed 4x4 block. + 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28, + // Shift left and insert two new columns in transposed 4x4 block. + 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29, + // Shift left and insert three new columns in transposed 4x4 block. + 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30 + // clang-format on +}; + static inline int16x4_t convolve4_4_2d_h(uint8x16_t samples, const int8x8_t x_filter, const int32x4_t correction, @@ -668,3 +680,610 @@ subpel_x_qn, conv_params); } } + +static inline int16x4_t convolve8_4_y(const int8x16_t s0, const int8x16_t s1, + const int8x8_t filters) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + int32x4_t sum = vdotq_lane_s32(acc, s0, filters, 0); + sum = vdotq_lane_s32(sum, s1, filters, 1); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint16x8_t convolve8_8_y(const int8x16_t s0_lo, + const int8x16_t s0_hi, + const int8x16_t s1_lo, + const int8x16_t s1_hi, + const int8x8_t filters, + const int16x8_t round_offset) { + // The sample range transform and permutation are performed by the caller. + // Accumulate into 128 << FILTER_BITS to account for range transform. + // (- 1 since we halved the filters.) + const int32x4_t acc = vdupq_n_s32(128 << (FILTER_BITS - 1)); + + int32x4_t sum0123 = vdotq_lane_s32(acc, s0_lo, filters, 0); + sum0123 = vdotq_lane_s32(sum0123, s1_lo, filters, 1); + + int32x4_t sum4567 = vdotq_lane_s32(acc, s0_hi, filters, 0); + sum4567 = vdotq_lane_s32(sum4567, s1_hi, filters, 1); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + + // We halved the filter values so -1 from right shift. + return vreinterpretq_u16_s16( + vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1)); +} + +static inline void dist_wtd_convolve_y_8tap_neon_dotprod( + const uint8_t *src_ptr, int src_stride, int w, int h, + const int16_t *y_filter_ptr, ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + src_ptr += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + int8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + int16x8_t d01 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1); + int16x8_t d23 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1); + + store_u16x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d01)); + store_u16x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d23)); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + s += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); + + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + src_ptr += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + int8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_dist_wtd_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), + d01, d23, fwd_offset, bck_offset, + round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + s += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); + + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, + bck_offset, round_offset_vec, &d0_u8, &d1_u8, + &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_avg_neon_dotprod( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + src_ptr += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + int8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_s8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_s8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_s8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_s8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(src_ptr, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_s8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_basic_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), d01, + d23, round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t t0, t1, t2, t3, t4, t5, t6; + load_u8_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); + s += 7 * src_stride; + + // Transform sample range to [-128, 127] for 8-bit signed dot product. + int8x8_t s0 = vreinterpret_s8_u8(vsub_u8(t0, vdup_n_u8(128))); + int8x8_t s1 = vreinterpret_s8_u8(vsub_u8(t1, vdup_n_u8(128))); + int8x8_t s2 = vreinterpret_s8_u8(vsub_u8(t2, vdup_n_u8(128))); + int8x8_t s3 = vreinterpret_s8_u8(vsub_u8(t3, vdup_n_u8(128))); + int8x8_t s4 = vreinterpret_s8_u8(vsub_u8(t4, vdup_n_u8(128))); + int8x8_t s5 = vreinterpret_s8_u8(vsub_u8(t5, vdup_n_u8(128))); + int8x8_t s6 = vreinterpret_s8_u8(vsub_u8(t6, vdup_n_u8(128))); + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_s8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_s8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_s8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_s8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t t7, t8, t9, tA; + load_u8_8x4(s, src_stride, &t7, &t8, &t9, &tA); + + int8x8_t s7 = vreinterpret_s8_u8(vsub_u8(t7, vdup_n_u8(128))); + int8x8_t s8 = vreinterpret_s8_u8(vsub_u8(t8, vdup_n_u8(128))); + int8x8_t s9 = vreinterpret_s8_u8(vsub_u8(t9, vdup_n_u8(128))); + int8x8_t sA = vreinterpret_s8_u8(vsub_u8(tA, vdup_n_u8(128))); + + int8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_s8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + int8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_s8(samples_LUT_lo, merge_block_tbl.val[2]); + + int8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_s8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +void av1_dist_wtd_convolve_y_neon_dotprod( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_dotprod( + src - 3 * src_stride, src_stride, dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } else { + dist_wtd_convolve_y_8tap_avg_neon_dotprod( + src - 3 * src_stride, src_stride, dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } + } else { + dist_wtd_convolve_y_8tap_neon_dotprod(src - 3 * src_stride, src_stride, w, + h, y_filter_ptr, conv_params); + } +}
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 1d14bd1..4d88276 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -611,7 +611,7 @@ specialize qw/av1_dist_wtd_convolve_2d ssse3 avx2 neon neon_dotprod neon_i8mm/; specialize qw/av1_dist_wtd_convolve_2d_copy sse2 avx2 neon/; specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon neon_dotprod neon_i8mm/; - specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon/; + specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon neon_dotprod/; if(aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { specialize qw/av1_highbd_dist_wtd_convolve_2d sse4_1 avx2 neon sve2/; specialize qw/av1_highbd_dist_wtd_convolve_x sse4_1 avx2 neon sve2/;
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc index 5392e5b..0ef4e1e 100644 --- a/test/av1_convolve_test.cc +++ b/test/av1_convolve_test.cc
@@ -2092,6 +2092,12 @@ BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon)); #endif +#if HAVE_NEON_DOTPROD +INSTANTIATE_TEST_SUITE_P( + NEON_DOTPROD, AV1ConvolveYCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon_dotprod)); +#endif + #if CONFIG_AV1_HIGHBITDEPTH ///////////////////////////////////////////////// // Compound convolve-y functions (high bit-depth)