Add Neon implementation of aom_convolve8_vert
Add an Armv8.0 Neon path for aom_convolve8_vert, and a corresponding
set of unit tests.
The Neon code added in this change is mostly a port of the Armv8.0
Neon implementation of the 'vpx_convolve8_vert_neon' function in
libvpx.[1]
[1] https://github.com/webmproject/libvpx/blob/main/vpx_dsp/arm/vpx_convolve8_neon.c
Change-Id: Ic8ecbaef418fb8110459acdf1877e0285163e280
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index c936328..604bb5a 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -473,7 +473,7 @@
specialize qw/aom_convolve_copy neon sse2 avx2/;
specialize qw/aom_convolve8_horiz neon sse2 ssse3/, "$avx2_ssse3";
-specialize qw/aom_convolve8_vert sse2 ssse3/, "$avx2_ssse3";
+specialize qw/aom_convolve8_vert neon sse2 ssse3/, "$avx2_ssse3";
add_proto qw/void aom_scaled_2d/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h";
specialize qw/aom_scaled_2d ssse3 neon/;
diff --git a/aom_dsp/arm/aom_convolve8_neon.c b/aom_dsp/arm/aom_convolve8_neon.c
index b5fb14d..8136c06 100644
--- a/aom_dsp/arm/aom_convolve8_neon.c
+++ b/aom_dsp/arm/aom_convolve8_neon.c
@@ -249,3 +249,119 @@
}
}
}
+
+void aom_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
+ uint8_t *dst, ptrdiff_t dst_stride,
+ const int16_t *filter_x, int x_step_q4,
+ const int16_t *filter_y, int y_step_q4, int w,
+ int h) {
+ const InterpKernel *const filters_y = get_filter_base(filter_y);
+ const int y0_q4 = get_filter_offset(filter_y, filters_y);
+ const int16x8_t filters = vld1q_s16(filters_y[y0_q4]);
+
+ assert((intptr_t)dst % 4 == 0);
+ assert(dst_stride % 4 == 0);
+
+ (void)filter_x;
+ (void)x_step_q4;
+ (void)y_step_q4;
+
+ src -= ((SUBPEL_TAPS / 2) - 1) * src_stride;
+
+ if (w == 4) {
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, d01, d23;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, d0, d1, d2, d3;
+
+ load_u8_8x7(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+ s0 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ s1 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ s2 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ s3 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+ s4 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t4)));
+ s5 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t5)));
+ s6 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t6)));
+
+ src += 7 * src_stride;
+
+ do {
+ load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
+ s7 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t0)));
+ s8 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t1)));
+ s9 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t2)));
+ s10 = vget_low_s16(vreinterpretq_s16_u16(vmovl_u8(t3)));
+
+ d0 = convolve8_4(s0, s1, s2, s3, s4, s5, s6, s7, filters);
+ d1 = convolve8_4(s1, s2, s3, s4, s5, s6, s7, s8, filters);
+ d2 = convolve8_4(s2, s3, s4, s5, s6, s7, s8, s9, filters);
+ d3 = convolve8_4(s3, s4, s5, s6, s7, s8, s9, s10, filters);
+ d01 = vqrshrun_n_s16(vcombine_s16(d0, d1), FILTER_BITS);
+ d23 = vqrshrun_n_s16(vcombine_s16(d2, d3), FILTER_BITS);
+
+ store_u8_4x1(dst + 0 * dst_stride, d01, 0);
+ store_u8_4x1(dst + 1 * dst_stride, d01, 1);
+ store_u8_4x1(dst + 2 * dst_stride, d23, 0);
+ store_u8_4x1(dst + 3 * dst_stride, d23, 1);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ uint8x8_t t0, t1, t2, t3, t4, t5, t6, d0, d1, d2, d3;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ int height;
+ const uint8_t *s;
+ uint8_t *d;
+
+ do {
+ load_u8_8x7(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+ s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ s2 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ s3 = vreinterpretq_s16_u16(vmovl_u8(t3));
+ s4 = vreinterpretq_s16_u16(vmovl_u8(t4));
+ s5 = vreinterpretq_s16_u16(vmovl_u8(t5));
+ s6 = vreinterpretq_s16_u16(vmovl_u8(t6));
+
+ height = h;
+ s = src + 7 * src_stride;
+ d = dst;
+
+ do {
+ load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
+ s7 = vreinterpretq_s16_u16(vmovl_u8(t0));
+ s8 = vreinterpretq_s16_u16(vmovl_u8(t1));
+ s9 = vreinterpretq_s16_u16(vmovl_u8(t2));
+ s10 = vreinterpretq_s16_u16(vmovl_u8(t3));
+
+ d0 = convolve8_8(s0, s1, s2, s3, s4, s5, s6, s7, filters);
+ d1 = convolve8_8(s1, s2, s3, s4, s5, s6, s7, s8, filters);
+ d2 = convolve8_8(s2, s3, s4, s5, s6, s7, s8, s9, filters);
+ d3 = convolve8_8(s3, s4, s5, s6, s7, s8, s9, s10, filters);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
diff --git a/test/convolve_test.cc b/test/convolve_test.cc
index 53cf1d6..da12378 100644
--- a/test/convolve_test.cc
+++ b/test/convolve_test.cc
@@ -868,7 +868,7 @@
#if HAVE_NEON
const ConvolveFunctions convolve8_neon(aom_convolve8_horiz_neon,
- aom_convolve8_vert_c, 0);
+ aom_convolve8_vert_neon, 0);
const ConvolveParam kArray_Convolve8_neon[] = { ALL_SIZES(convolve8_neon) };
INSTANTIATE_TEST_SUITE_P(NEON, LowbdConvolveTest,
