[NEON] highbd implementation of av1_highbd_convolve_y_sr_neon 1/3
Approximately 6.1x faster. This is the first part of the 8-tap filter
implementation. Helper functions have been moved to own header.
Change-Id: Iedaee35580b7c7506aebec10c47a714bfccad60f
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h
index 3410fb8..d27cff6 100644
--- a/aom_dsp/arm/mem_neon.h
+++ b/aom_dsp/arm/mem_neon.h
@@ -174,6 +174,26 @@
s += p;
}
+static INLINE void load_u16_4x7(const uint16_t *s, ptrdiff_t p,
+ uint16x4_t *const s0, uint16x4_t *const s1,
+ uint16x4_t *const s2, uint16x4_t *const s3,
+ uint16x4_t *const s4, uint16x4_t *const s5,
+ uint16x4_t *const s6) {
+ *s0 = vld1_u16(s);
+ s += p;
+ *s1 = vld1_u16(s);
+ s += p;
+ *s2 = vld1_u16(s);
+ s += p;
+ *s3 = vld1_u16(s);
+ s += p;
+ *s4 = vld1_u16(s);
+ s += p;
+ *s5 = vld1_u16(s);
+ s += p;
+ *s6 = vld1_u16(s);
+}
+
static INLINE void load_u16_8x4(const uint16_t *s, const ptrdiff_t p,
uint16x8_t *const s0, uint16x8_t *const s1,
uint16x8_t *const s2, uint16x8_t *const s3) {
@@ -393,6 +413,13 @@
vst1_u16(s, s3);
}
+static INLINE void store_u16_8x2(uint16_t *s, ptrdiff_t dst_stride,
+ const uint16x8_t s0, const uint16x8_t s1) {
+ vst1q_u16(s, s0);
+ s += dst_stride;
+ vst1q_u16(s, s1);
+}
+
static INLINE void store_u16_8x4(uint16_t *s, ptrdiff_t dst_stride,
const uint16x8_t s0, const uint16x8_t s1,
const uint16x8_t s2, const uint16x8_t s3) {
@@ -445,6 +472,14 @@
do { \
vst1_lane_s32((int32_t *)(s), vreinterpret_s32_s16(s0), lane); \
} while (0)
+#define store_u16_2x1(s, s0, lane) \
+ do { \
+ vst1_lane_u32((uint32_t *)(s), vreinterpret_u32_u16(s0), lane); \
+ } while (0)
+#define store_u16q_2x1(s, s0, lane) \
+ do { \
+ vst1q_lane_u32((uint32_t *)(s), vreinterpretq_u32_u16(s0), lane); \
+ } while (0)
static INLINE void store_s16_8x4(int16_t *s, ptrdiff_t dst_stride,
const int16x8_t s0, const int16x8_t s1,
@@ -540,6 +575,26 @@
*s7 = vld1q_s16(s);
}
+static INLINE void load_u16_8x7(const uint16_t *s, ptrdiff_t p,
+ uint16x8_t *const s0, uint16x8_t *const s1,
+ uint16x8_t *const s2, uint16x8_t *const s3,
+ uint16x8_t *const s4, uint16x8_t *const s5,
+ uint16x8_t *const s6) {
+ *s0 = vld1q_u16(s);
+ s += p;
+ *s1 = vld1q_u16(s);
+ s += p;
+ *s2 = vld1q_u16(s);
+ s += p;
+ *s3 = vld1q_u16(s);
+ s += p;
+ *s4 = vld1q_u16(s);
+ s += p;
+ *s5 = vld1q_u16(s);
+ s += p;
+ *s6 = vld1q_u16(s);
+}
+
static INLINE void load_s16_8x7(const int16_t *s, ptrdiff_t p,
int16x8_t *const s0, int16x8_t *const s1,
int16x8_t *const s2, int16x8_t *const s3,
diff --git a/av1/av1.cmake b/av1/av1.cmake
index ece0771..992b5c3 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -443,6 +443,9 @@
"${AOM_ROOT}/av1/common/x86/highbd_wiener_convolve_avx2.c"
"${AOM_ROOT}/av1/common/x86/highbd_warp_affine_avx2.c")
+ list(APPEND AOM_AV1_COMMON_INTRIN_NEON
+ "${AOM_ROOT}/av1/common/arm/highbd_convolve_neon.c")
+
list(APPEND AOM_AV1_ENCODER_INTRIN_SSE2
"${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_sse2.c"
"${AOM_ROOT}/av1/encoder/x86/highbd_temporal_filter_sse2.c")
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
new file mode 100644
index 0000000..3af834f
--- /dev/null
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -0,0 +1,177 @@
+/*
+ *
+ * Copyright (c) 2023, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <assert.h>
+#include <arm_neon.h>
+
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/arm/mem_neon.h"
+#include "aom_dsp/arm/transpose_neon.h"
+#include "aom_ports/mem.h"
+#include "av1/common/convolve.h"
+#include "av1/common/filter.h"
+#include "av1/common/arm/highbd_convolve_neon.h"
+
+void av1_highbd_convolve_y_sr_neon(const uint16_t *src, int src_stride,
+ uint16_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_y_qn, int bd) {
+ const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
+ if (y_filter_taps > 8) {
+ av1_highbd_convolve_y_sr_c(src, src_stride, dst, dst_stride, w, h,
+ filter_params_y, subpel_y_qn, bd);
+ return;
+ }
+
+ const int vert_offset = filter_params_y->taps / 2 - 1;
+ const uint16x8_t max = vdupq_n_u16(bd == 10 ? 1023 : (bd == 12 ? 4095 : 255));
+
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+
+ src -= vert_offset * src_stride;
+
+ if (w <= 4) {
+ uint16x4_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ uint16x4_t d0, d1, d2, d3;
+ uint16x8_t d01, d23;
+
+ const uint16_t *s = src;
+ uint16_t *d = dst;
+
+ load_u16_4x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+ s0 = vreinterpret_s16_u16(t0);
+ s1 = vreinterpret_s16_u16(t1);
+ s2 = vreinterpret_s16_u16(t2);
+ s3 = vreinterpret_s16_u16(t3);
+ s4 = vreinterpret_s16_u16(t4);
+ s5 = vreinterpret_s16_u16(t5);
+ s6 = vreinterpret_s16_u16(t6);
+
+ s += 7 * src_stride;
+
+ do {
+ load_u16_4x4(s, src_stride, &t7, &t8, &t9, &t10);
+ s7 = vreinterpret_s16_u16(t7);
+ s8 = vreinterpret_s16_u16(t8);
+ s9 = vreinterpret_s16_u16(t9);
+ s10 = vreinterpret_s16_u16(t10);
+
+ d0 = highbd_convolve8_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+ d1 = highbd_convolve8_4_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8, y_filter);
+ d2 = highbd_convolve8_4_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9, y_filter);
+ d3 =
+ highbd_convolve8_4_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10, y_filter);
+
+ d01 = vcombine_u16(d0, d1);
+ d23 = vcombine_u16(d2, d3);
+
+ d01 = vminq_u16(d01, max);
+ d23 = vminq_u16(d23, max);
+
+ if (w == 2) {
+ store_u16q_2x1(d + 0 * dst_stride, d01, 0);
+ store_u16q_2x1(d + 1 * dst_stride, d01, 2);
+ if (h != 2) {
+ store_u16q_2x1(d + 2 * dst_stride, d23, 0);
+ store_u16q_2x1(d + 3 * dst_stride, d23, 2);
+ }
+ } else {
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ if (h != 2) {
+ vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
+ vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
+ }
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ int height;
+ uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ uint16x8_t d0, d1, d2, d3;
+ do {
+ const uint16_t *s = src;
+ uint16_t *d = dst;
+
+ load_u16_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
+ s0 = vreinterpretq_s16_u16(t0);
+ s1 = vreinterpretq_s16_u16(t1);
+ s2 = vreinterpretq_s16_u16(t2);
+ s3 = vreinterpretq_s16_u16(t3);
+ s4 = vreinterpretq_s16_u16(t4);
+ s5 = vreinterpretq_s16_u16(t5);
+ s6 = vreinterpretq_s16_u16(t6);
+
+ s += 7 * src_stride;
+ height = h;
+
+ do {
+ load_u16_8x4(s, src_stride, &t7, &t8, &t9, &t10);
+ s7 = vreinterpretq_s16_u16(t7);
+ s8 = vreinterpretq_s16_u16(t8);
+ s9 = vreinterpretq_s16_u16(t9);
+ s10 = vreinterpretq_s16_u16(t10);
+
+ d0 = highbd_convolve8_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter);
+ d1 = highbd_convolve8_8_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter);
+ d2 = highbd_convolve8_8_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter);
+ d3 = highbd_convolve8_8_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ if (h != 2) {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+ } else {
+ store_u16_8x2(d, dst_stride, d0, d1);
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+ src += 8;
+ dst += 8;
+ w -= 8;
+ } while (w > 0);
+ }
+}
diff --git a/av1/common/arm/highbd_convolve_neon.h b/av1/common/arm/highbd_convolve_neon.h
new file mode 100644
index 0000000..c683cf0
--- /dev/null
+++ b/av1/common/arm/highbd_convolve_neon.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2023, Alliance for Open Media. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef AOM_AV1_COMMON_ARM_HIGHBD_CONVOLVE_NEON_H_
+#define AOM_AV1_COMMON_ARM_HIGHBD_CONVOLVE_NEON_H_
+
+#include <arm_neon.h>
+
+static INLINE int32x4_t highbd_convolve8_4_s32(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_lo, 0);
+ sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1);
+ sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2);
+ sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3);
+ sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0);
+ sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1);
+ sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2);
+ sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3);
+
+ return sum;
+}
+
+static INLINE uint16x4_t highbd_convolve8_4_s32_s16(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter) {
+ int32x4_t sum =
+ highbd_convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE void highbd_convolve8_8_s32(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
+ int32x4_t *sum0, int32x4_t *sum1) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ *sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 0);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s1), y_filter_lo, 1);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s2), y_filter_lo, 2);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s3), y_filter_lo, 3);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s4), y_filter_hi, 0);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s5), y_filter_hi, 1);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s6), y_filter_hi, 2);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s7), y_filter_hi, 3);
+
+ *sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 0);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s1), y_filter_lo, 1);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s2), y_filter_lo, 2);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s3), y_filter_lo, 3);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s4), y_filter_hi, 0);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s5), y_filter_hi, 1);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s6), y_filter_hi, 2);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s7), y_filter_hi, 3);
+}
+
+static INLINE uint16x8_t highbd_convolve8_8_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter) {
+ int32x4_t sum0;
+ int32x4_t sum1;
+ highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, &sum0,
+ &sum1);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
+#endif // AOM_AV1_COMMON_ARM_HIGHBD_CONVOLVE_NEON_H_
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index cec8f13..845fd15 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -597,7 +597,7 @@
specialize qw/av1_highbd_dist_wtd_convolve_2d_copy sse4_1 avx2/;
specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2/;
specialize qw/av1_highbd_convolve_x_sr ssse3 avx2/;
- specialize qw/av1_highbd_convolve_y_sr ssse3 avx2/;
+ specialize qw/av1_highbd_convolve_y_sr ssse3 avx2 neon/;
specialize qw/av1_highbd_convolve_2d_scale sse4_1/;
}
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc
index 12edfac..16ac687 100644
--- a/test/av1_convolve_test.cc
+++ b/test/av1_convolve_test.cc
@@ -735,6 +735,11 @@
BuildHighbdParams(av1_highbd_convolve_y_sr_avx2));
#endif
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1ConvolveYHighbdTest,
+ BuildHighbdParams(av1_highbd_convolve_y_sr_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
//////////////////////////////////////////////////////////////
