Add 4-tap specialisation to aom_convolve8_vert_neon Add a specialized Neon path for 4-tap filters in aom_convolve8_vert_neon, and use it for the neon_dotprod and neon_i8mm versions as well. This gives between 40% and 50% uplift compared to using the 8-tap path. Change-Id: I93e48a62d851af5ff0c6f8015cb91f687c970802
diff --git a/aom_dsp/arm/aom_convolve8_neon.c b/aom_dsp/arm/aom_convolve8_neon.c index 5665b5e..43aef54 100644 --- a/aom_dsp/arm/aom_convolve8_neon.c +++ b/aom_dsp/arm/aom_convolve8_neon.c
@@ -243,18 +243,6 @@ return sum; } -static INLINE uint8x8_t convolve4_8(const int16x8_t s0, const int16x8_t s1, - const int16x8_t s2, const int16x8_t s3, - const int16x4_t filter) { - int16x8_t sum = vmulq_lane_s16(s0, filter, 0); - sum = vmlaq_lane_s16(sum, s1, filter, 1); - sum = vmlaq_lane_s16(sum, s2, filter, 2); - sum = vmlaq_lane_s16(sum, s3, filter, 3); - - // We halved the filter values so -1 from right shift. - return vqrshrun_n_s16(sum, FILTER_BITS - 1); -} - static INLINE void convolve8_horiz_4tap_neon(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, @@ -368,22 +356,13 @@ } } -void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const int16_t *filter_x, int x_step_q4, - const int16_t *filter_y, int y_step_q4, int w, - int h) { +static INLINE void convolve8_vert_8tap_neon(const uint8_t *src, + ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, + const int16_t *filter_y, int w, + int h) { const int16x8_t filter = vld1q_s16(filter_y); - assert((intptr_t)dst % 4 == 0); - assert(dst_stride % 4 == 0); - - (void)filter_x; - (void)x_step_q4; - (void)y_step_q4; - - src -= ((SUBPEL_TAPS / 2) - 1) * src_stride; - if (w == 4) { uint8x8_t t0, t1, t2, t3, t4, t5, t6; load_u8_8x7(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); @@ -478,3 +457,25 @@ } while (w != 0); } } + +void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + assert((intptr_t)dst % 4 == 0); + assert(dst_stride % 4 == 0); + + (void)filter_x; + (void)x_step_q4; + (void)y_step_q4; + + src -= ((SUBPEL_TAPS / 2) - 1) * src_stride; + + if (get_filter_taps_convolve8(filter_y) <= 4) { + convolve8_vert_4tap_neon(src + 2 * src_stride, src_stride, dst, dst_stride, + filter_y, w, h); + } else { + convolve8_vert_8tap_neon(src, src_stride, dst, dst_stride, filter_y, w, h); + } +}
diff --git a/aom_dsp/arm/aom_convolve8_neon.h b/aom_dsp/arm/aom_convolve8_neon.h index 0aebc6d..83fbd0a 100644 --- a/aom_dsp/arm/aom_convolve8_neon.h +++ b/aom_dsp/arm/aom_convolve8_neon.h
@@ -103,4 +103,102 @@ } } +static INLINE uint8x8_t convolve4_8(const int16x8_t s0, const int16x8_t s1, + const int16x8_t s2, const int16x8_t s3, + const int16x4_t filter) { + int16x8_t sum = vmulq_lane_s16(s0, filter, 0); + sum = vmlaq_lane_s16(sum, s1, filter, 1); + sum = vmlaq_lane_s16(sum, s2, filter, 2); + sum = vmlaq_lane_s16(sum, s3, filter, 3); + + // We halved the filter values so -1 from right shift. + return vqrshrun_n_s16(sum, FILTER_BITS - 1); +} + +static INLINE void convolve8_vert_4tap_neon(const uint8_t *src, + ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, + const int16_t *filter_y, int w, + int h) { + // All filter values are even, halve to reduce intermediate precision + // requirements. + const int16x4_t filter = vshr_n_s16(vld1_s16(filter_y + 2), 1); + + if (w == 4) { + uint8x8_t t01 = load_unaligned_u8(src + 0 * src_stride, (int)src_stride); + uint8x8_t t12 = load_unaligned_u8(src + 1 * src_stride, (int)src_stride); + + int16x8_t s01 = vreinterpretq_s16_u16(vmovl_u8(t01)); + int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t12)); + + src += 2 * src_stride; + + do { + uint8x8_t t23 = load_unaligned_u8(src + 0 * src_stride, (int)src_stride); + uint8x8_t t34 = load_unaligned_u8(src + 1 * src_stride, (int)src_stride); + uint8x8_t t45 = load_unaligned_u8(src + 2 * src_stride, (int)src_stride); + uint8x8_t t56 = load_unaligned_u8(src + 3 * src_stride, (int)src_stride); + + int16x8_t s23 = vreinterpretq_s16_u16(vmovl_u8(t23)); + int16x8_t s34 = vreinterpretq_s16_u16(vmovl_u8(t34)); + int16x8_t s45 = vreinterpretq_s16_u16(vmovl_u8(t45)); + int16x8_t s56 = vreinterpretq_s16_u16(vmovl_u8(t56)); + + uint8x8_t d01 = convolve4_8(s01, s12, s23, s34, filter); + uint8x8_t d23 = convolve4_8(s23, s34, s45, s56, filter); + + store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); + store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); + + s01 = s45; + s12 = s56; + + src += 4 * src_stride; + dst += 4 * dst_stride; + h -= 4; + } while (h != 0); + } else { + do { + uint8x8_t t0, t1, t2; + load_u8_8x3(src, src_stride, &t0, &t1, &t2); + + int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0)); + int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1)); + int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2)); + + int height = h; + const uint8_t *s = src + 3 * src_stride; + uint8_t *d = dst; + + do { + uint8x8_t t3; + load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); + + int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t0)); + int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t1)); + int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t2)); + int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t3)); + + uint8x8_t d0 = convolve4_8(s0, s1, s2, s3, filter); + uint8x8_t d1 = convolve4_8(s1, s2, s3, s4, filter); + uint8x8_t d2 = convolve4_8(s2, s3, s4, s5, filter); + uint8x8_t d3 = convolve4_8(s3, s4, s5, s6, filter); + + store_u8_8x4(d, dst_stride, d0, d1, d2, d3); + + s0 = s4; + s1 = s5; + s2 = s6; + + s += 4 * src_stride; + d += 4 * dst_stride; + height -= 4; + } while (height != 0); + src += 8; + dst += 8; + w -= 8; + } while (w != 0); + } +} + #endif // AOM_AOM_DSP_ARM_AOM_CONVOLVE8_NEON_H_
diff --git a/aom_dsp/arm/aom_convolve8_neon_dotprod.c b/aom_dsp/arm/aom_convolve8_neon_dotprod.c index f49d33f..4d47d86 100644 --- a/aom_dsp/arm/aom_convolve8_neon_dotprod.c +++ b/aom_dsp/arm/aom_convolve8_neon_dotprod.c
@@ -370,24 +370,13 @@ return vqrshrun_n_s16(sum, FILTER_BITS); } -void aom_convolve8_vert_neon_dotprod(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const int16_t *filter_x, int x_step_q4, - const int16_t *filter_y, int y_step_q4, - int w, int h) { +static INLINE void convolve8_vert_8tap_neon_dotprod( + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_y, int w, int h) { const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y)); const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); int8x16x2_t samples_LUT; - assert((intptr_t)dst % 4 == 0); - assert(dst_stride % 4 == 0); - - (void)filter_x; - (void)x_step_q4; - (void)y_step_q4; - - src -= ((SUBPEL_TAPS / 2) - 1) * src_stride; - if (w == 4) { uint8x8_t t0, t1, t2, t3, t4, t5, t6; load_u8_8x7(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6); @@ -536,3 +525,26 @@ } while (w != 0); } } + +void aom_convolve8_vert_neon_dotprod(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h) { + assert((intptr_t)dst % 4 == 0); + assert(dst_stride % 4 == 0); + + (void)filter_x; + (void)x_step_q4; + (void)y_step_q4; + + src -= ((SUBPEL_TAPS / 2) - 1) * src_stride; + + if (get_filter_taps_convolve8(filter_y) <= 4) { + convolve8_vert_4tap_neon(src + 2 * src_stride, src_stride, dst, dst_stride, + filter_y, w, h); + } else { + convolve8_vert_8tap_neon_dotprod(src, src_stride, dst, dst_stride, filter_y, + w, h); + } +}
diff --git a/aom_dsp/arm/aom_convolve8_neon_i8mm.c b/aom_dsp/arm/aom_convolve8_neon_i8mm.c index 9727639..21a4551 100644 --- a/aom_dsp/arm/aom_convolve8_neon_i8mm.c +++ b/aom_dsp/arm/aom_convolve8_neon_i8mm.c
@@ -340,24 +340,13 @@ return vqrshrun_n_s16(sum, FILTER_BITS); } -void aom_convolve8_vert_neon_i8mm(const uint8_t *src, ptrdiff_t src_stride, - uint8_t *dst, ptrdiff_t dst_stride, - const int16_t *filter_x, int x_step_q4, - const int16_t *filter_y, int y_step_q4, int w, - int h) { +static INLINE void convolve8_vert_8tap_neon_i8mm( + const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const int16_t *filter_y, int w, int h) { const int8x8_t filter = vmovn_s16(vld1q_s16(filter_y)); const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(kDotProdMergeBlockTbl); uint8x16x2_t samples_LUT; - assert((intptr_t)dst % 4 == 0); - assert(dst_stride % 4 == 0); - - (void)filter_x; - (void)x_step_q4; - (void)y_step_q4; - - src -= ((SUBPEL_TAPS / 2) - 1) * src_stride; - if (w == 4) { uint8x8_t s0, s1, s2, s3, s4, s5, s6; load_u8_8x7(src, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); @@ -478,3 +467,26 @@ } while (w != 0); } } + +void aom_convolve8_vert_neon_i8mm(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, int w, + int h) { + assert((intptr_t)dst % 4 == 0); + assert(dst_stride % 4 == 0); + + (void)filter_x; + (void)x_step_q4; + (void)y_step_q4; + + src -= ((SUBPEL_TAPS / 2) - 1) * src_stride; + + if (get_filter_taps_convolve8(filter_y) <= 4) { + convolve8_vert_4tap_neon(src + 2 * src_stride, src_stride, dst, dst_stride, + filter_y, w, h); + } else { + convolve8_vert_8tap_neon_i8mm(src, src_stride, dst, dst_stride, filter_y, w, + h); + } +}
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h index 32a462a..ba18700 100644 --- a/aom_dsp/arm/mem_neon.h +++ b/aom_dsp/arm/mem_neon.h
@@ -174,6 +174,16 @@ *s3 = vld1_u8(s); } +static INLINE void load_u8_8x3(const uint8_t *s, const ptrdiff_t p, + uint8x8_t *const s0, uint8x8_t *const s1, + uint8x8_t *const s2) { + *s0 = vld1_u8(s); + s += p; + *s1 = vld1_u8(s); + s += p; + *s2 = vld1_u8(s); +} + static INLINE void load_u16_4x4(const uint16_t *s, const ptrdiff_t p, uint16x4_t *const s0, uint16x4_t *const s1, uint16x4_t *const s2, uint16x4_t *const s3) {