[NEON] Optimize av1_highbd_convolve_2d_scale_neon().
CPU usage drops from 64% to 48% and in superres-mode=1 encoding
time is reduced by 38% (total).
Change-Id: Ia8dc93a5f31a446beb164a59bf4a883b3b764a0e
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
index 5a4b9c8..e0dc9b4 100644
--- a/av1/common/arm/highbd_convolve_neon.c
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -663,16 +663,16 @@
do {
load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
- d0 = highbd_convolve8_4_rs_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ d0 = highbd_convolve8_4_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
y_filter, round1_shift_s32, offset_s32,
correction_s32);
- d1 = highbd_convolve8_4_rs_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
+ d1 = highbd_convolve8_4_sr_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
y_filter, round1_shift_s32, offset_s32,
correction_s32);
- d2 = highbd_convolve8_4_rs_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
+ d2 = highbd_convolve8_4_sr_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
y_filter, round1_shift_s32, offset_s32,
correction_s32);
- d3 = highbd_convolve8_4_rs_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
+ d3 = highbd_convolve8_4_sr_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
y_filter, round1_shift_s32, offset_s32,
correction_s32);
@@ -723,16 +723,16 @@
do {
load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
- d0 = highbd_convolve8_8_rs_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ d0 = highbd_convolve8_8_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
y_filter, round1_shift_s32,
offset_s32, correction_s32);
- d1 = highbd_convolve8_8_rs_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
+ d1 = highbd_convolve8_8_sr_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
y_filter, round1_shift_s32,
offset_s32, correction_s32);
- d2 = highbd_convolve8_8_rs_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
+ d2 = highbd_convolve8_8_sr_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
y_filter, round1_shift_s32,
offset_s32, correction_s32);
- d3 = highbd_convolve8_8_rs_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
+ d3 = highbd_convolve8_8_sr_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
y_filter, round1_shift_s32,
offset_s32, correction_s32);
@@ -792,16 +792,16 @@
do {
load_s16_4x4(s, src_stride, &s11, &s12, &s13, &s14);
- d0 = highbd_convolve12_y_4_rs_s32_s16(
+ d0 = highbd_convolve12_y_4_sr_s32_s16(
s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d1 = highbd_convolve12_y_4_rs_s32_s16(
+ d1 = highbd_convolve12_y_4_sr_s32_s16(
s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d2 = highbd_convolve12_y_4_rs_s32_s16(
+ d2 = highbd_convolve12_y_4_sr_s32_s16(
s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d3 = highbd_convolve12_y_4_rs_s32_s16(
+ d3 = highbd_convolve12_y_4_sr_s32_s16(
s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
@@ -858,16 +858,16 @@
do {
load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
- d0 = highbd_convolve12_y_8_rs_s32_s16(
+ d0 = highbd_convolve12_y_8_sr_s32_s16(
s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d1 = highbd_convolve12_y_8_rs_s32_s16(
+ d1 = highbd_convolve12_y_8_sr_s32_s16(
s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d2 = highbd_convolve12_y_8_rs_s32_s16(
+ d2 = highbd_convolve12_y_8_sr_s32_s16(
s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
- d3 = highbd_convolve12_y_8_rs_s32_s16(
+ d3 = highbd_convolve12_y_8_sr_s32_s16(
s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, y_filter_0_7,
y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
@@ -1124,3 +1124,484 @@
bd, y_offset_initial, y_offset_correction_s32);
}
}
+
+static INLINE void highbd_convolve_2d_x_scale_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int subpel_x_qn, const int x_step_qn,
+ const InterpFilterParams *filter_params, ConvolveParams *conv_params,
+ const int offset) {
+ const uint32x4_t idx = { 0, 1, 2, 3 };
+ const uint32x4_t subpel_mask = vdupq_n_u32(SCALE_SUBPEL_MASK);
+ const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+
+ if (w <= 4) {
+ int height = h;
+ int16x8_t s0, s1, s2, s3;
+ uint16x4_t d0;
+
+ uint16_t *d = dst_ptr;
+
+ do {
+ int x_qn = subpel_x_qn;
+
+ // Load 4 src vectors at a time, they might be the same, but we have to
+ // calculate the indices anyway. Doing it in SIMD and then storing the
+ // indices is faster than having to calculate the expression
+ // &src_ptr[((x_qn + 0*x_step_qn) >> SCALE_SUBPEL_BITS)] 4 times
+ // Ideally this should be a gather using the indices, but NEON does not
+ // have that, so have to emulate
+ const uint32x4_t xqn_idx = vmlaq_n_u32(vdupq_n_u32(x_qn), idx, x_step_qn);
+ // We have to multiply x2 to get the actual pointer as sizeof(uint16_t) =
+ // 2
+ const uint32x4_t src_idx_u32 =
+ vshlq_n_u32(vshrq_n_u32(xqn_idx, SCALE_SUBPEL_BITS), 1);
+#if defined(__aarch64__)
+ uint64x2_t src4[2];
+ src4[0] = vaddw_u32(vdupq_n_u64((const uint64_t)src_ptr),
+ vget_low_u32(src_idx_u32));
+ src4[1] = vaddw_u32(vdupq_n_u64((const uint64_t)src_ptr),
+ vget_high_u32(src_idx_u32));
+ int16_t *src4_ptr[4];
+ uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
+ vst1q_u64(tmp_ptr, src4[0]);
+ vst1q_u64(tmp_ptr + 2, src4[1]);
+#else
+ uint32x4_t src4;
+ src4 = vaddq_u32(vdupq_n_u32((const uint32_t)src_ptr), src_idx_u32);
+ int16_t *src4_ptr[4];
+ uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
+ vst1q_u32(tmp_ptr, src4);
+#endif // defined(__aarch64__)
+ // Same for the filter vectors
+ const int32x4_t filter_idx_s32 = vreinterpretq_s32_u32(
+ vshrq_n_u32(vandq_u32(xqn_idx, subpel_mask), SCALE_EXTRA_BITS));
+ int32_t x_filter4_idx[4];
+ vst1q_s32(x_filter4_idx, filter_idx_s32);
+ const int16_t *x_filter4_ptr[4];
+
+ // Load source
+ s0 = vld1q_s16(src4_ptr[0]);
+ s1 = vld1q_s16(src4_ptr[1]);
+ s2 = vld1q_s16(src4_ptr[2]);
+ s3 = vld1q_s16(src4_ptr[3]);
+
+ // We could easily do this using SIMD as well instead of calling the
+ // inline function 4 times.
+ x_filter4_ptr[0] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[0]);
+ x_filter4_ptr[1] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[1]);
+ x_filter4_ptr[2] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[2]);
+ x_filter4_ptr[3] =
+ av1_get_interp_filter_subpel_kernel(filter_params, x_filter4_idx[3]);
+
+ // Actually load the filters
+ const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
+ const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
+ const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
+ const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+
+ // Group low and high parts and transpose
+ int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
+ vget_low_s16(x_filter1),
+ vget_low_s16(x_filter2),
+ vget_low_s16(x_filter3) };
+ int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
+ vget_high_s16(x_filter1),
+ vget_high_s16(x_filter2),
+ vget_high_s16(x_filter3) };
+ transpose_u16_4x4((uint16x4_t *)filters_lo);
+ transpose_u16_4x4((uint16x4_t *)filters_hi);
+
+ // Run the 2D Scale convolution
+ d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+
+ if (w == 2) {
+ store_u16_2x1(d + 0 * dst_stride, d0, 0);
+ } else {
+ vst1_u16(d + 0 * dst_stride, d0);
+ }
+
+ src_ptr += src_stride;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ } else {
+ int height = h;
+ int16x8_t s0, s1, s2, s3;
+ uint16x4_t d0;
+
+ do {
+ int width = w;
+ int x_qn = subpel_x_qn;
+ uint16_t *d = dst_ptr;
+ const uint16_t *s = src_ptr;
+
+ do {
+ // Load 4 src vectors at a time, they might be the same, but we have to
+ // calculate the indices anyway. Doing it in SIMD and then storing the
+ // indices is faster than having to calculate the expression
+ // &src_ptr[((x_qn + 0*x_step_qn) >> SCALE_SUBPEL_BITS)] 4 times
+ // Ideally this should be a gather using the indices, but NEON does not
+ // have that, so have to emulate
+ const uint32x4_t xqn_idx =
+ vmlaq_n_u32(vdupq_n_u32(x_qn), idx, x_step_qn);
+ // We have to multiply x2 to get the actual pointer as sizeof(uint16_t)
+ // = 2
+ const uint32x4_t src_idx_u32 =
+ vshlq_n_u32(vshrq_n_u32(xqn_idx, SCALE_SUBPEL_BITS), 1);
+#if defined(__aarch64__)
+ uint64x2_t src4[2];
+ src4[0] = vaddw_u32(vdupq_n_u64((const uint64_t)s),
+ vget_low_u32(src_idx_u32));
+ src4[1] = vaddw_u32(vdupq_n_u64((const uint64_t)s),
+ vget_high_u32(src_idx_u32));
+ int16_t *src4_ptr[4];
+ uint64_t *tmp_ptr = (uint64_t *)&src4_ptr;
+ vst1q_u64(tmp_ptr, src4[0]);
+ vst1q_u64(tmp_ptr + 2, src4[1]);
+#else
+ uint32x4_t src4;
+ src4 = vaddq_u32(vdupq_n_u32((const uint32_t)s), src_idx_u32);
+ int16_t *src4_ptr[4];
+ uint32_t *tmp_ptr = (uint32_t *)&src4_ptr;
+ vst1q_u32(tmp_ptr, src4);
+#endif // defined(__aarch64__)
+ // Same for the filter vectors
+ const int32x4_t filter_idx_s32 = vreinterpretq_s32_u32(
+ vshrq_n_u32(vandq_u32(xqn_idx, subpel_mask), SCALE_EXTRA_BITS));
+ int32_t x_filter4_idx[4];
+ vst1q_s32(x_filter4_idx, filter_idx_s32);
+ const int16_t *x_filter4_ptr[4];
+
+ // Load source
+ s0 = vld1q_s16(src4_ptr[0]);
+ s1 = vld1q_s16(src4_ptr[1]);
+ s2 = vld1q_s16(src4_ptr[2]);
+ s3 = vld1q_s16(src4_ptr[3]);
+
+ // We could easily do this using SIMD as well instead of calling the
+ // inline function 4 times.
+ x_filter4_ptr[0] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[0]);
+ x_filter4_ptr[1] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[1]);
+ x_filter4_ptr[2] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[2]);
+ x_filter4_ptr[3] = av1_get_interp_filter_subpel_kernel(
+ filter_params, x_filter4_idx[3]);
+
+ // Actually load the filters
+ const int16x8_t x_filter0 = vld1q_s16(x_filter4_ptr[0]);
+ const int16x8_t x_filter1 = vld1q_s16(x_filter4_ptr[1]);
+ const int16x8_t x_filter2 = vld1q_s16(x_filter4_ptr[2]);
+ const int16x8_t x_filter3 = vld1q_s16(x_filter4_ptr[3]);
+
+ // Group low and high parts and transpose
+ int16x4_t filters_lo[] = { vget_low_s16(x_filter0),
+ vget_low_s16(x_filter1),
+ vget_low_s16(x_filter2),
+ vget_low_s16(x_filter3) };
+ int16x4_t filters_hi[] = { vget_high_s16(x_filter0),
+ vget_high_s16(x_filter1),
+ vget_high_s16(x_filter2),
+ vget_high_s16(x_filter3) };
+ transpose_u16_4x4((uint16x4_t *)filters_lo);
+ transpose_u16_4x4((uint16x4_t *)filters_hi);
+
+ // Run the 2D Scale X convolution
+ d0 = highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ s0, s1, s2, s3, filters_lo, filters_hi, shift_s32, offset_s32);
+
+ vst1_u16(d, d0);
+
+ x_qn += 4 * x_step_qn;
+ d += 4;
+ width -= 4;
+ } while (width > 0);
+
+ src_ptr += src_stride;
+ dst_ptr += dst_stride;
+ height--;
+ } while (height > 0);
+ }
+}
+
+static INLINE void highbd_convolve_2d_y_scale_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int subpel_y_qn, const int y_step_qn,
+ const InterpFilterParams *filter_params, const int round1_bits,
+ const int offset) {
+ const int32x4_t offset_s32 = vdupq_n_s32(1 << offset);
+
+ const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_bits);
+ if (w <= 4) {
+ int height = h;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7;
+ uint16x4_t d0;
+
+ uint16_t *d = dst_ptr;
+
+ int y_qn = subpel_y_qn;
+ do {
+ const int16_t *s =
+ (const int16_t *)&src_ptr[(y_qn >> SCALE_SUBPEL_BITS) * src_stride];
+
+ load_s16_4x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
+ const int16_t *y_filter_ptr =
+ av1_get_interp_filter_subpel_kernel(filter_params, y_filter_idx);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ d0 = highbd_convolve8_4_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, round1_shift_s32, offset_s32,
+ vdupq_n_s32(0));
+
+ if (w == 2) {
+ store_u16_2x1(d, d0, 0);
+ } else {
+ vst1_u16(d, d0);
+ }
+
+ y_qn += y_step_qn;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ } else {
+ int width = w;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
+ uint16x8_t d0;
+
+ do {
+ int height = h;
+ int y_qn = subpel_y_qn;
+
+ uint16_t *d = dst_ptr;
+
+ do {
+ const int16_t *s =
+ (const int16_t *)&src_ptr[(y_qn >> SCALE_SUBPEL_BITS) * src_stride];
+ load_s16_8x8(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
+
+ const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
+ const int16_t *y_filter_ptr =
+ av1_get_interp_filter_subpel_kernel(filter_params, y_filter_idx);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ d0 = highbd_convolve8_8_sr_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, round1_shift_s32,
+ offset_s32, vdupq_n_s32(0));
+ vst1q_u16(d, d0);
+
+ y_qn += y_step_qn;
+ d += dst_stride;
+ height--;
+ } while (height > 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ width -= 8;
+ } while (width > 0);
+ }
+}
+
+void av1_highbd_convolve_2d_scale_neon(
+ const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
+ int h, const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y, const int subpel_x_qn,
+ const int x_step_qn, const int subpel_y_qn, const int y_step_qn,
+ ConvolveParams *conv_params, int bd) {
+ uint16_t *im_block = (uint16_t *)aom_memalign(
+ 16, 2 * sizeof(uint16_t) * MAX_SB_SIZE * (MAX_SB_SIZE + MAX_FILTER_TAP));
+ if (!im_block) return;
+ uint16_t *im_block2 = (uint16_t *)aom_memalign(
+ 16, 2 * sizeof(uint16_t) * MAX_SB_SIZE * (MAX_SB_SIZE + MAX_FILTER_TAP));
+ if (!im_block2) {
+ aom_free(im_block); // free the first block and return.
+ return;
+ }
+
+ int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) +
+ filter_params_y->taps;
+ const int im_stride = MAX_SB_SIZE;
+ CONV_BUF_TYPE *dst16 = conv_params->dst;
+ const int dst16_stride = conv_params->dst_stride;
+ const int bits =
+ FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
+ assert(bits >= 0);
+
+ const int vert_offset = filter_params_y->taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const int x_offset_bits = (1 << (bd + FILTER_BITS - 1));
+ const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int y_offset_correction =
+ ((1 << (y_offset_bits - conv_params->round_1)) +
+ (1 << (y_offset_bits - conv_params->round_1 - 1)));
+
+ const int32x4_t final_shift_s32 = vdupq_n_s32(-bits);
+ const int16x4_t y_offset_correction_s16 = vdup_n_s16(y_offset_correction);
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ uint16x4_t fwd_offset_u16 = vdup_n_u16(conv_params->fwd_offset);
+ uint16x4_t bck_offset_u16 = vdup_n_u16(conv_params->bck_offset);
+
+ const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+
+ highbd_convolve_2d_x_scale_8tap_neon(
+ src_ptr, src_stride, im_block, im_stride, w, im_h, subpel_x_qn, x_step_qn,
+ filter_params_x, conv_params, x_offset_bits);
+ if (conv_params->is_compound && !conv_params->do_average) {
+ highbd_convolve_2d_y_scale_8tap_neon(
+ im_block, im_stride, dst16, dst16_stride, w, h, subpel_y_qn, y_step_qn,
+ filter_params_y, conv_params->round_1, y_offset_bits);
+ } else {
+ highbd_convolve_2d_y_scale_8tap_neon(
+ im_block, im_stride, im_block2, im_stride, w, h, subpel_y_qn, y_step_qn,
+ filter_params_y, conv_params->round_1, y_offset_bits);
+ }
+
+ // Do the compound averaging outside the loop, avoids branching within the
+ // main loop
+ if (conv_params->is_compound) {
+ if (conv_params->do_average) {
+ if (conv_params->use_dist_wtd_comp_avg) {
+ // Weighted averaging
+ if (w <= 4) {
+ for (int y = 0; y < h; ++y) {
+ const uint16x4_t s = vld1_u16(im_block2 + y * im_stride);
+ const uint16x4_t d16 = vld1_u16(dst16 + y * dst16_stride);
+ // We use vmull_u16/vmlal_u16 instead of of vmull_s16/vmlal_s16
+ // because the latter sign-extend and the values are non-negative.
+ // However, d0/d1 are signed-integers and we use vqmovun
+ // to do saturated narrowing to unsigned.
+ int32x4_t d0 =
+ vreinterpretq_s32_u32(vmull_u16(d16, fwd_offset_u16));
+ d0 = vreinterpretq_s32_u32(
+ vmlal_u16(vreinterpretq_u32_s32(d0), s, bck_offset_u16));
+ d0 = vshrq_n_s32(d0, DIST_PRECISION_BITS);
+ // Subtract round offset and convolve round
+ d0 = vqrshlq_s32(vsubw_s16(d0, y_offset_correction_s16),
+ final_shift_s32);
+ uint16x4_t d = vqmovun_s32(d0);
+ d = vmin_u16(d, vget_low_u16(max));
+ if (w == 2) {
+ store_u16_2x1(dst + y * dst_stride, d, 0);
+ } else {
+ vst1_u16(dst + y * dst_stride, d);
+ }
+ }
+ } else {
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; x += 8) {
+ const uint16x8_t s = vld1q_u16(im_block2 + y * im_stride + x);
+ const uint16x8_t d16 = vld1q_u16(dst16 + y * dst16_stride + x);
+ // We use vmull_u16/vmlal_u16 instead of of vmull_s16/vmlal_s16
+ // because the latter sign-extend and the values are non-negative.
+ // However, d0/d1 are signed-integers and we use vqmovun
+ // to do saturated narrowing to unsigned.
+ int32x4_t d0 = vreinterpretq_s32_u32(
+ vmull_u16(vget_low_u16(d16), fwd_offset_u16));
+ int32x4_t d1 = vreinterpretq_s32_u32(
+ vmull_u16(vget_high_u16(d16), fwd_offset_u16));
+ d0 = vreinterpretq_s32_u32(vmlal_u16(
+ vreinterpretq_u32_s32(d0), vget_low_u16(s), bck_offset_u16));
+ d1 = vreinterpretq_s32_u32(vmlal_u16(
+ vreinterpretq_u32_s32(d1), vget_high_u16(s), bck_offset_u16));
+ d0 = vshrq_n_s32(d0, DIST_PRECISION_BITS);
+ d1 = vshrq_n_s32(d1, DIST_PRECISION_BITS);
+ d0 = vqrshlq_s32(vsubw_s16(d0, y_offset_correction_s16),
+ final_shift_s32);
+ d1 = vqrshlq_s32(vsubw_s16(d1, y_offset_correction_s16),
+ final_shift_s32);
+ uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst + y * dst_stride + x, d01);
+ }
+ }
+ }
+ } else {
+ if (w <= 4) {
+ for (int y = 0; y < h; ++y) {
+ const uint16x4_t s = vld1_u16(im_block2 + y * im_stride);
+ const uint16x4_t d16 = vld1_u16(dst16 + y * dst16_stride);
+ int32x4_t s_s32 = vreinterpretq_s32_u32(vmovl_u16(s));
+ int32x4_t d16_s32 = vreinterpretq_s32_u32(vmovl_u16(d16));
+ int32x4_t d0 = vhaddq_s32(s_s32, d16_s32);
+ d0 = vsubw_s16(d0, y_offset_correction_s16);
+ d0 = vqrshlq_s32(d0, final_shift_s32);
+ uint16x4_t d = vqmovun_s32(d0);
+ d = vmin_u16(d, vget_low_u16(max));
+ if (w == 2) {
+ store_u16_2x1(dst + y * dst_stride, d, 0);
+ } else {
+ vst1_u16(dst + y * dst_stride, d);
+ }
+ }
+ } else {
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; x += 8) {
+ const uint16x8_t s = vld1q_u16(im_block2 + y * im_stride + x);
+ const uint16x8_t d16 = vld1q_u16(dst16 + y * dst16_stride + x);
+ int32x4_t s_lo =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(s)));
+ int32x4_t s_hi =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(s)));
+ int32x4_t d16_lo =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(d16)));
+ int32x4_t d16_hi =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(d16)));
+ int32x4_t d0 = vhaddq_s32(s_lo, d16_lo);
+ int32x4_t d1 = vhaddq_s32(s_hi, d16_hi);
+ d0 = vsubw_s16(d0, y_offset_correction_s16);
+ d1 = vsubw_s16(d1, y_offset_correction_s16);
+ d0 = vqrshlq_s32(d0, final_shift_s32);
+ d1 = vqrshlq_s32(d1, final_shift_s32);
+ uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst + y * dst_stride + x, d01);
+ }
+ }
+ }
+ }
+ }
+ } else {
+ // No compound averaging
+ if (w <= 4) {
+ for (int y = 0; y < h; ++y) {
+ // Subtract round offset and convolve round
+ const int16x4_t s =
+ vld1_s16((const int16_t *)(im_block2) + y * im_stride);
+ const int32x4_t d0 =
+ vqrshlq_s32(vsubl_s16(s, y_offset_correction_s16), final_shift_s32);
+ uint16x4_t d = vqmovun_s32(d0);
+ d = vmin_u16(d, vget_low_u16(max));
+ if (w == 2) {
+ store_u16_2x1(dst + y * dst_stride, d, 0);
+ } else {
+ vst1_u16(dst + y * dst_stride, d);
+ }
+ }
+ } else {
+ for (int y = 0; y < h; ++y) {
+ for (int x = 0; x < w; x += 8) {
+ // Subtract round offset and convolve round
+ const int16x8_t s =
+ vld1q_s16((const int16_t *)(im_block2) + y * im_stride + x);
+ const int32x4_t d0 =
+ vqrshlq_s32(vsubl_s16(vget_low_s16(s), y_offset_correction_s16),
+ final_shift_s32);
+ const int32x4_t d1 =
+ vqrshlq_s32(vsubl_s16(vget_high_s16(s), y_offset_correction_s16),
+ final_shift_s32);
+ uint16x8_t d01 = vcombine_u16(vqmovun_s32(d0), vqmovun_s32(d1));
+ d01 = vminq_u16(d01, max);
+ vst1q_u16(dst + y * dst_stride + x, d01);
+ }
+ }
+ }
+ }
+ aom_free(im_block);
+ aom_free(im_block2);
+}
diff --git a/av1/common/arm/highbd_convolve_neon.h b/av1/common/arm/highbd_convolve_neon.h
index 7934463..ed33be1 100644
--- a/av1/common/arm/highbd_convolve_neon.h
+++ b/av1/common/arm/highbd_convolve_neon.h
@@ -109,7 +109,7 @@
}
// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x4_t highbd_convolve8_4_rs_s32_s16(
+static INLINE uint16x4_t highbd_convolve8_4_sr_s32_s16(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
@@ -164,7 +164,7 @@
}
// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x8_t highbd_convolve8_8_rs_s32_s16(
+static INLINE uint16x8_t highbd_convolve8_8_sr_s32_s16(
const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
@@ -222,7 +222,7 @@
}
// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x4_t highbd_convolve12_y_4_rs_s32_s16(
+static INLINE uint16x4_t highbd_convolve12_y_4_sr_s32_s16(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
@@ -292,7 +292,7 @@
}
// Like above but also perform round shifting and subtract correction term
-static INLINE uint16x8_t highbd_convolve12_y_8_rs_s32_s16(
+static INLINE uint16x8_t highbd_convolve12_y_8_sr_s32_s16(
const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
@@ -439,4 +439,40 @@
return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
}
+static INLINE int32x4_t highbd_convolve8_2d_scale_horiz4x8_s32(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x4_t *filters_lo,
+ const int16x4_t *filters_hi, const int32x4_t offset) {
+ int16x4_t s_lo[] = { vget_low_s16(s0), vget_low_s16(s1), vget_low_s16(s2),
+ vget_low_s16(s3) };
+ int16x4_t s_hi[] = { vget_high_s16(s0), vget_high_s16(s1), vget_high_s16(s2),
+ vget_high_s16(s3) };
+
+ transpose_u16_4x4((uint16x4_t *)s_lo);
+ transpose_u16_4x4((uint16x4_t *)s_hi);
+
+ int32x4_t sum = vmlal_s16(offset, s_lo[0], filters_lo[0]);
+ sum = vmlal_s16(sum, s_lo[1], filters_lo[1]);
+ sum = vmlal_s16(sum, s_lo[2], filters_lo[2]);
+ sum = vmlal_s16(sum, s_lo[3], filters_lo[3]);
+ sum = vmlal_s16(sum, s_hi[0], filters_hi[0]);
+ sum = vmlal_s16(sum, s_hi[1], filters_hi[1]);
+ sum = vmlal_s16(sum, s_hi[2], filters_hi[2]);
+ sum = vmlal_s16(sum, s_hi[3], filters_hi[3]);
+
+ return sum;
+}
+
+static INLINE uint16x4_t highbd_convolve8_2d_scale_horiz4x8_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x4_t *filters_lo,
+ const int16x4_t *filters_hi, const int32x4_t shift_s32,
+ const int32x4_t offset) {
+ int32x4_t sum = highbd_convolve8_2d_scale_horiz4x8_s32(
+ s0, s1, s2, s3, filters_lo, filters_hi, offset);
+
+ sum = vqrshlq_s32(sum, shift_s32);
+ return vqmovun_s32(sum);
+}
+
#endif // AOM_AV1_COMMON_ARM_HIGHBD_CONVOLVE_NEON_H_
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 9f9961a..31eb440 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -603,7 +603,7 @@
specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2 neon/;
specialize qw/av1_highbd_convolve_x_sr ssse3 avx2 neon/;
specialize qw/av1_highbd_convolve_y_sr ssse3 avx2 neon/;
- specialize qw/av1_highbd_convolve_2d_scale sse4_1/;
+ specialize qw/av1_highbd_convolve_2d_scale sse4_1 neon/;
}
# INTRA_EDGE functions
diff --git a/test/av1_convolve_scale_test.cc b/test/av1_convolve_scale_test.cc
index 3f35025..00f0a09 100644
--- a/test/av1_convolve_scale_test.cc
+++ b/test/av1_convolve_scale_test.cc
@@ -455,11 +455,20 @@
TEST_P(LowBDConvolveScaleTest, DISABLED_Speed) { SpeedTest(); }
INSTANTIATE_TEST_SUITE_P(
+ C, LowBDConvolveScaleTest,
+ ::testing::Combine(::testing::Values(av1_convolve_2d_scale_c),
+ ::testing::ValuesIn(kBlockDim),
+ ::testing::ValuesIn(kNTaps), ::testing::ValuesIn(kNTaps),
+ ::testing::Bool()));
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(
SSE4_1, LowBDConvolveScaleTest,
::testing::Combine(::testing::Values(av1_convolve_2d_scale_sse4_1),
::testing::ValuesIn(kBlockDim),
::testing::ValuesIn(kNTaps), ::testing::ValuesIn(kNTaps),
::testing::Bool()));
+#endif // HAVE_SSE4_1
#if CONFIG_AV1_HIGHBITDEPTH
typedef void (*HighbdConvolveFunc)(const uint16_t *src, int src_stride,
@@ -522,10 +531,30 @@
TEST_P(HighBDConvolveScaleTest, DISABLED_Speed) { SpeedTest(); }
INSTANTIATE_TEST_SUITE_P(
+ C, HighBDConvolveScaleTest,
+ ::testing::Combine(::testing::Values(av1_highbd_convolve_2d_scale_c),
+ ::testing::ValuesIn(kBlockDim),
+ ::testing::ValuesIn(kNTaps), ::testing::ValuesIn(kNTaps),
+ ::testing::Bool(), ::testing::ValuesIn(kBDs)));
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_SUITE_P(
SSE4_1, HighBDConvolveScaleTest,
::testing::Combine(::testing::Values(av1_highbd_convolve_2d_scale_sse4_1),
::testing::ValuesIn(kBlockDim),
::testing::ValuesIn(kNTaps), ::testing::ValuesIn(kNTaps),
::testing::Bool(), ::testing::ValuesIn(kBDs)));
+#endif // HAVE_SSE4_1
+
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, HighBDConvolveScaleTest,
+ ::testing::Combine(::testing::Values(av1_highbd_convolve_2d_scale_neon),
+ ::testing::ValuesIn(kBlockDim),
+ ::testing::ValuesIn(kNTaps), ::testing::ValuesIn(kNTaps),
+ ::testing::Bool(), ::testing::ValuesIn(kBDs)));
+
+#endif // HAVE_NEON
+
#endif // CONFIG_AV1_HIGHBITDEPTH
} // namespace
diff --git a/test/test.cmake b/test/test.cmake
index 8e5cb87..51b4e47 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -354,6 +354,11 @@
endif()
+ if(HAVE_NEON)
+ list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES
+ "${AOM_ROOT}/test/av1_convolve_scale_test.cc")
+ endif()
+
if(HAVE_SSE4_2 OR HAVE_ARM_CRC32)
list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES "${AOM_ROOT}/test/hash_test.cc")
endif()
