Add 4-tap merged impl of av1_convolve_2d_sr_neon_i8mm
Merge the horizontal and vertical passes of av1_convolve_2d_sr_neon_i8mm
for 4-tap filters, avoiding the use of an intermediate buffer. This
gives around 10% uplift over the split implementation.
Change-Id: I4dcdd8521a709649449f5dd737bb7c6cdc0c2b6f
diff --git a/av1/common/arm/convolve_neon_i8mm.c b/av1/common/arm/convolve_neon_i8mm.c
index 8f54b64..cd989cb 100644
--- a/av1/common/arm/convolve_neon_i8mm.c
+++ b/av1/common/arm/convolve_neon_i8mm.c
@@ -1056,6 +1056,130 @@
} while (w != 0);
}
+static INLINE void convolve_2d_sr_4tap_neon_i8mm(const uint8_t *src,
+ int src_stride, uint8_t *dst,
+ int dst_stride, int w, int h,
+ const int16_t *x_filter_ptr,
+ const int16_t *y_filter_ptr) {
+ const int bd = 8;
+ const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1));
+
+ const int16x4_t y_filter = vld1_s16(y_filter_ptr + 2);
+ const int16x4_t x_filter_s16 = vld1_s16(x_filter_ptr + 2);
+ // All 4-tap and bilinear filter values are even, so halve them to reduce
+ // intermediate precision requirements.
+ const int8x8_t x_filter =
+ vshrn_n_s16(vcombine_s16(x_filter_s16, vdup_n_s16(0)), 1);
+
+ // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding
+ // shifts - which are generally faster than rounding shifts on modern CPUs.
+ // Halve the total because we halved the filter values.
+ const int32x4_t horiz_const = vdupq_n_s32(
+ ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))) / 2);
+
+ if (w == 4) {
+ const uint8x16_t permute_tbl = vld1q_u8(kDotProdPermuteTbl);
+
+ uint8x16_t h_s0, h_s1, h_s2;
+ load_u8_16x3(src, src_stride, &h_s0, &h_s1, &h_s2);
+
+ int16x4_t v_s0 = convolve4_4_2d_h(h_s0, x_filter, permute_tbl, horiz_const);
+ int16x4_t v_s1 = convolve4_4_2d_h(h_s1, x_filter, permute_tbl, horiz_const);
+ int16x4_t v_s2 = convolve4_4_2d_h(h_s2, x_filter, permute_tbl, horiz_const);
+
+ src += 3 * src_stride;
+
+ do {
+ uint8x16_t h_s3, h_s4, h_s5, h_s6;
+ load_u8_16x4(src, src_stride, &h_s3, &h_s4, &h_s5, &h_s6);
+
+ int16x4_t v_s3 =
+ convolve4_4_2d_h(h_s3, x_filter, permute_tbl, horiz_const);
+ int16x4_t v_s4 =
+ convolve4_4_2d_h(h_s4, x_filter, permute_tbl, horiz_const);
+ int16x4_t v_s5 =
+ convolve4_4_2d_h(h_s5, x_filter, permute_tbl, horiz_const);
+ int16x4_t v_s6 =
+ convolve4_4_2d_h(h_s6, x_filter, permute_tbl, horiz_const);
+
+ int16x4_t d0 = convolve4_4_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter);
+ int16x4_t d1 = convolve4_4_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter);
+ int16x4_t d2 = convolve4_4_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter);
+ int16x4_t d3 = convolve4_4_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter);
+
+ uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), vert_const));
+ uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), vert_const));
+
+ store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01);
+ store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23);
+
+ v_s0 = v_s4;
+ v_s1 = v_s5;
+ v_s2 = v_s6;
+
+ src += 4 * src_stride;
+ dst += 4 * dst_stride;
+ h -= 4;
+ } while (h != 0);
+ } else {
+ const uint8x16x2_t permute_tbl = vld1q_u8_x2(kDotProdPermuteTbl);
+
+ do {
+ int height = h;
+ const uint8_t *s = src;
+ uint8_t *d = dst;
+
+ uint8x16_t h_s0, h_s1, h_s2;
+ load_u8_16x3(src, src_stride, &h_s0, &h_s1, &h_s2);
+
+ int16x8_t v_s0 =
+ convolve4_8_2d_h(h_s0, x_filter, permute_tbl, horiz_const);
+ int16x8_t v_s1 =
+ convolve4_8_2d_h(h_s1, x_filter, permute_tbl, horiz_const);
+ int16x8_t v_s2 =
+ convolve4_8_2d_h(h_s2, x_filter, permute_tbl, horiz_const);
+
+ s += 3 * src_stride;
+
+ do {
+ uint8x16_t h_s3, h_s4, h_s5, h_s6;
+ load_u8_16x4(s, src_stride, &h_s3, &h_s4, &h_s5, &h_s6);
+
+ int16x8_t v_s3 =
+ convolve4_8_2d_h(h_s3, x_filter, permute_tbl, horiz_const);
+ int16x8_t v_s4 =
+ convolve4_8_2d_h(h_s4, x_filter, permute_tbl, horiz_const);
+ int16x8_t v_s5 =
+ convolve4_8_2d_h(h_s5, x_filter, permute_tbl, horiz_const);
+ int16x8_t v_s6 =
+ convolve4_8_2d_h(h_s6, x_filter, permute_tbl, horiz_const);
+
+ uint8x8_t d0 =
+ convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter, vert_const);
+ uint8x8_t d1 =
+ convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter, vert_const);
+ uint8x8_t d2 =
+ convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter, vert_const);
+ uint8x8_t d3 =
+ convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter, vert_const);
+
+ store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
+
+ v_s0 = v_s4;
+ v_s1 = v_s5;
+ v_s2 = v_s6;
+
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height != 0);
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w != 0);
+ }
+}
+
void av1_convolve_2d_sr_neon_i8mm(const uint8_t *src, int src_stride,
uint8_t *dst, int dst_stride, int w, int h,
const InterpFilterParams *filter_params_x,
@@ -1108,6 +1232,12 @@
return;
}
+ if (y_filter_taps <= 4 && x_filter_taps <= 4) {
+ convolve_2d_sr_4tap_neon_i8mm(src_ptr + 2, src_stride, dst, dst_stride, w,
+ h, x_filter_ptr, y_filter_ptr);
+ return;
+ }
+
if (x_filter_taps <= 4) {
convolve_2d_sr_horiz_4tap_neon_i8mm(src_ptr + 2, src_stride, im_block,
im_stride, w, im_h, x_filter_ptr);
