Add Armv8.6 I8MM USDOT path for av1_dist_wtd_convolve_y Implement Armv8.6 I8MM USDOT path for av1_dist_wtd_convolve_y and add the associated unit tests. Change-Id: I1d7f92729c1fb7a72e8e444897ed7d739592df1d
diff --git a/av1/common/arm/compound_convolve_neon_i8mm.c b/av1/common/arm/compound_convolve_neon_i8mm.c index fa91308..79e4fba 100644 --- a/av1/common/arm/compound_convolve_neon_i8mm.c +++ b/av1/common/arm/compound_convolve_neon_i8mm.c
@@ -13,6 +13,7 @@ #include <assert.h> #include "aom_dsp/arm/mem_neon.h" +#include "aom_dsp/arm/transpose_neon.h" #include "av1/common/arm/compound_convolve_neon.h" #include "config/aom_config.h" #include "config/av1_rtcd.h" @@ -43,6 +44,17 @@ // clang-format on }; +DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { + // clang-format off + // Shift left and insert new last column in transposed 4x4 block. + 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28, + // Shift left and insert two new columns in transposed 4x4 block. + 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29, + // Shift left and insert three new columns in transposed 4x4 block. + 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30 + // clang-format on +}; + static inline int16x4_t convolve6_4_2d_h(uint8x16_t samples, const int8x16_t x_filter, const uint8x16_t permute_tbl, @@ -967,3 +979,523 @@ x_filter_ptr); } } + +static inline int16x4_t convolve8_4_y(const uint8x16_t s0, const uint8x16_t s1, + const int8x8_t filters) { + int32x4_t sum = vusdotq_lane_s32(vdupq_n_s32(0), s0, filters, 0); + sum = vusdotq_lane_s32(sum, s1, filters, 1); + + // Further narrowing and packing is performed by the caller. + return vmovn_s32(sum); +} + +static inline uint16x8_t convolve8_8_y(const uint8x16_t s0_lo, + const uint8x16_t s0_hi, + const uint8x16_t s1_lo, + const uint8x16_t s1_hi, + const int8x8_t filters, + const int16x8_t round_offset) { + int32x4_t sum0123 = vusdotq_lane_s32(vdupq_n_s32(0), s0_lo, filters, 0); + sum0123 = vusdotq_lane_s32(sum0123, s1_lo, filters, 1); + + int32x4_t sum4567 = vusdotq_lane_s32(vdupq_n_s32(0), s0_hi, filters, 0); + sum4567 = vusdotq_lane_s32(sum4567, s1_hi, filters, 1); + + // Narrow and re-pack. + int16x8_t sum = vcombine_s16(vmovn_s32(sum0123), vmovn_s32(sum4567)); + + // We halved the filter values so -1 from right shift. + return vreinterpretq_u16_s16( + vrsraq_n_s16(round_offset, sum, ROUND0_BITS - 1)); +} + +static inline void dist_wtd_convolve_y_8tap_neon_i8mm( + const uint8_t *src_ptr, int src_stride, int w, int h, + const int16_t *y_filter_ptr, ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + src_ptr += 7 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // (see horizontal case) required before computing the dot product. + uint8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + int16x8_t d01 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1); + int16x8_t d23 = + vrsraq_n_s16(round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1); + + store_u16x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d01)); + store_u16x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, + vreinterpretq_u16_s16(d23)); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + s += 7 * src_stride; + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); + + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + store_u16_8x4(d, dst_stride, d0, d1, d2, d3); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_i8mm( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + src_ptr += 7 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // (see horizontal case) required before computing the dot product. + uint8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_dist_wtd_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), + d01, d23, fwd_offset, bck_offset, + round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + s += 7 * src_stride; + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); + + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_dist_wtd_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, fwd_offset, + bck_offset, round_offset_vec, &d0_u8, &d1_u8, + &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +static inline void dist_wtd_convolve_y_8tap_avg_neon_i8mm( + const uint8_t *src_ptr, int src_stride, uint8_t *dst8_ptr, + const int dst8_stride, int w, int h, const int16_t *y_filter_ptr, + ConvolveParams *conv_params) { + const int bd = 8; + const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; + const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); + const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; + + // Filter values are even, so halve to reduce intermediate precision reqs. + const int8x8_t filter = vshrn_n_s16(vld1q_s16(y_filter_ptr), 1); + + const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); + + if (w == 4) { + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + src_ptr += 7 * src_stride; + + // This operation combines a conventional transpose and the sample permute + // (see horizontal case) required before computing the dot product. + uint8x16_t s0123, s1234, s2345, s3456; + transpose_concat_elems_u8_4x4(s0, s1, s2, s3, &s0123); + transpose_concat_elems_u8_4x4(s1, s2, s3, s4, &s1234); + transpose_concat_elems_u8_4x4(s2, s3, s4, s5, &s2345); + transpose_concat_elems_u8_4x4(s3, s4, s5, s6, &s3456); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(src_ptr, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567, s5678, s6789, s789A; + transpose_concat_elems_u8_4x4(s7, s8, s9, sA, &s789A); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT = { { s3456, s789A } }; + s4567 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[0]); + s5678 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[1]); + s6789 = vqtbl2q_u8(samples_LUT, merge_block_tbl.val[2]); + + int16x4_t d0 = convolve8_4_y(s0123, s4567, filter); + int16x4_t d1 = convolve8_4_y(s1234, s5678, filter); + int16x4_t d2 = convolve8_4_y(s2345, s6789, filter); + int16x4_t d3 = convolve8_4_y(s3456, s789A, filter); + + // We halved the filter values so -1 from right shift. + uint16x8_t d01 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d0, d1), ROUND0_BITS - 1)); + uint16x8_t d23 = vreinterpretq_u16_s16(vrsraq_n_s16( + round_offset_vec, vcombine_s16(d2, d3), ROUND0_BITS - 1)); + + uint16x4_t dd0, dd1, dd2, dd3; + load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8; + compute_basic_avg_8x2(vcombine_u16(dd0, dd1), vcombine_u16(dd2, dd3), d01, + d23, round_offset_vec, &d0_u8, &d1_u8); + + store_u8x4_strided_x2(dst8_ptr + 0 * dst8_stride, dst8_stride, d0_u8); + store_u8x4_strided_x2(dst8_ptr + 2 * dst8_stride, dst8_stride, d1_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123 = s4567; + s1234 = s5678; + s2345 = s6789; + s3456 = s789A; + + src_ptr += 4 * src_stride; + dst_ptr += 4 * dst_stride; + dst8_ptr += 4 * dst8_stride; + h -= 4; + } while (h != 0); + } else { + do { + int height = h; + const uint8_t *s = src_ptr; + CONV_BUF_TYPE *d = dst_ptr; + uint8_t *d_u8 = dst8_ptr; + + uint8x8_t s0, s1, s2, s3, s4, s5, s6; + load_u8_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); + s += 7 * src_stride; + + // This operation combines a conventional transpose and the sample + // permute (see horizontal case) required before computing the dot + // product. + uint8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi, + s3456_lo, s3456_hi; + transpose_concat_elems_u8_8x4(s0, s1, s2, s3, &s0123_lo, &s0123_hi); + transpose_concat_elems_u8_8x4(s1, s2, s3, s4, &s1234_lo, &s1234_hi); + transpose_concat_elems_u8_8x4(s2, s3, s4, s5, &s2345_lo, &s2345_hi); + transpose_concat_elems_u8_8x4(s3, s4, s5, s6, &s3456_lo, &s3456_hi); + + do { + uint8x8_t s7, s8, s9, sA; + load_u8_8x4(s, src_stride, &s7, &s8, &s9, &sA); + + uint8x16_t s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo, s6789_hi, + s789A_lo, s789A_hi; + transpose_concat_elems_u8_8x4(s7, s8, s9, sA, &s789A_lo, &s789A_hi); + + // Merge new data into block from previous iteration. + uint8x16x2_t samples_LUT_lo = { { s3456_lo, s789A_lo } }; + s4567_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[0]); + s5678_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[1]); + s6789_lo = vqtbl2q_u8(samples_LUT_lo, merge_block_tbl.val[2]); + + uint8x16x2_t samples_LUT_hi = { { s3456_hi, s789A_hi } }; + s4567_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[0]); + s5678_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[1]); + s6789_hi = vqtbl2q_u8(samples_LUT_hi, merge_block_tbl.val[2]); + + uint16x8_t d0 = convolve8_8_y(s0123_lo, s0123_hi, s4567_lo, s4567_hi, + filter, round_offset_vec); + uint16x8_t d1 = convolve8_8_y(s1234_lo, s1234_hi, s5678_lo, s5678_hi, + filter, round_offset_vec); + uint16x8_t d2 = convolve8_8_y(s2345_lo, s2345_hi, s6789_lo, s6789_hi, + filter, round_offset_vec); + uint16x8_t d3 = convolve8_8_y(s3456_lo, s3456_hi, s789A_lo, s789A_hi, + filter, round_offset_vec); + + uint16x8_t dd0, dd1, dd2, dd3; + load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); + + uint8x8_t d0_u8, d1_u8, d2_u8, d3_u8; + compute_basic_avg_8x4(dd0, dd1, dd2, dd3, d0, d1, d2, d3, + round_offset_vec, &d0_u8, &d1_u8, &d2_u8, &d3_u8); + + store_u8_8x4(d_u8, dst8_stride, d0_u8, d1_u8, d2_u8, d3_u8); + + // Prepare block for next iteration - re-using as much as possible. + // Shuffle everything up four rows. + s0123_lo = s4567_lo; + s0123_hi = s4567_hi; + s1234_lo = s5678_lo; + s1234_hi = s5678_hi; + s2345_lo = s6789_lo; + s2345_hi = s6789_hi; + s3456_lo = s789A_lo; + s3456_hi = s789A_hi; + + s += 4 * src_stride; + d += 4 * dst_stride; + d_u8 += 4 * dst8_stride; + height -= 4; + } while (height != 0); + src_ptr += 8; + dst_ptr += 8; + dst8_ptr += 8; + w -= 8; + } while (w != 0); + } +} + +void av1_dist_wtd_convolve_y_neon_i8mm( + const uint8_t *src, int src_stride, uint8_t *dst8, int dst8_stride, int w, + int h, const InterpFilterParams *filter_params_y, const int subpel_y_qn, + ConvolveParams *conv_params) { + assert(w % 4 == 0); + assert(h % 4 == 0); + + const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel( + filter_params_y, subpel_y_qn & SUBPEL_MASK); + + if (conv_params->do_average) { + if (UNLIKELY(conv_params->use_dist_wtd_comp_avg)) { + dist_wtd_convolve_y_8tap_dist_wtd_avg_neon_i8mm( + src - 3 * src_stride, src_stride, dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } else { + dist_wtd_convolve_y_8tap_avg_neon_i8mm(src - 3 * src_stride, src_stride, + dst8, dst8_stride, w, h, + y_filter_ptr, conv_params); + } + } else { + dist_wtd_convolve_y_8tap_neon_i8mm(src - 3 * src_stride, src_stride, w, h, + y_filter_ptr, conv_params); + } +}
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 4d88276..c10d74d 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -611,7 +611,7 @@ specialize qw/av1_dist_wtd_convolve_2d ssse3 avx2 neon neon_dotprod neon_i8mm/; specialize qw/av1_dist_wtd_convolve_2d_copy sse2 avx2 neon/; specialize qw/av1_dist_wtd_convolve_x sse2 avx2 neon neon_dotprod neon_i8mm/; - specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon neon_dotprod/; + specialize qw/av1_dist_wtd_convolve_y sse2 avx2 neon neon_dotprod neon_i8mm/; if(aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { specialize qw/av1_highbd_dist_wtd_convolve_2d sse4_1 avx2 neon sve2/; specialize qw/av1_highbd_dist_wtd_convolve_x sse4_1 avx2 neon sve2/;
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc index 0ef4e1e..7acec93 100644 --- a/test/av1_convolve_test.cc +++ b/test/av1_convolve_test.cc
@@ -2098,6 +2098,12 @@ BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon_dotprod)); #endif +#if HAVE_NEON_I8MM +INSTANTIATE_TEST_SUITE_P( + NEON_I8MM, AV1ConvolveYCompoundTest, + BuildLowbdLumaParams(av1_dist_wtd_convolve_y_neon_i8mm)); +#endif + #if CONFIG_AV1_HIGHBITDEPTH ///////////////////////////////////////////////// // Compound convolve-y functions (high bit-depth)