Add Neon implementation of av1_wedge_sse_from_residuals
Add an Arm Neon implementation of aom_wedge_sse_from_residuals and
use it instead of the scalar C implementation.
Also add test coverage for the new Neon implementation.
Change-Id: If332330f086366825a9369b3279142d748a2d7ed
diff --git a/av1/av1.cmake b/av1/av1.cmake
index c017427..72dc3e7 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -364,7 +364,8 @@
"${AOM_ROOT}/av1/encoder/arm/neon/encodetxb_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/hybrid_fwd_txfm_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c"
- "${AOM_ROOT}/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c")
+ "${AOM_ROOT}/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/wedge_utils_neon.c")
list(APPEND AOM_AV1_ENCODER_INTRIN_MSA
"${AOM_ROOT}/av1/encoder/mips/msa/error_msa.c"
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index c9e87e3..97912be 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -442,7 +442,7 @@
specialize qw/av1_txb_init_levels sse4_1 avx2 neon/;
add_proto qw/uint64_t av1_wedge_sse_from_residuals/, "const int16_t *r1, const int16_t *d, const uint8_t *m, int N";
- specialize qw/av1_wedge_sse_from_residuals sse2 avx2/;
+ specialize qw/av1_wedge_sse_from_residuals sse2 avx2 neon/;
add_proto qw/int8_t av1_wedge_sign_from_residuals/, "const int16_t *ds, const uint8_t *m, int N, int64_t limit";
specialize qw/av1_wedge_sign_from_residuals sse2 avx2/;
add_proto qw/void av1_wedge_compute_delta_squares/, "int16_t *d, const int16_t *a, const int16_t *b, int N";
diff --git a/av1/encoder/arm/neon/wedge_utils_neon.c b/av1/encoder/arm/neon/wedge_utils_neon.c
new file mode 100644
index 0000000..54d8d19
--- /dev/null
+++ b/av1/encoder/arm/neon/wedge_utils_neon.c
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2022, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <assert.h>
+
+#include "aom_dsp/arm/sum_neon.h"
+#include "av1/common/reconinter.h"
+
+#define MAX_MASK_VALUE (1 << WEDGE_WEIGHT_BITS)
+
+/**
+ * See av1_wedge_sse_from_residuals_c for details of the parameters and
+ * computation.
+ */
+uint64_t av1_wedge_sse_from_residuals_neon(const int16_t *r1, const int16_t *d,
+ const uint8_t *m, int N) {
+ assert(N % 64 == 0);
+
+ uint64x2_t v_csse[2] = { vdupq_n_u64(0), vdupq_n_u64(0) };
+
+ int i = 0;
+ do {
+ int32x4_t sum[4];
+ int32x4_t sse[2];
+ int16x4_t sum_s16[4];
+
+ const int16x8_t r1_l = vld1q_s16(r1 + i);
+ const int16x8_t r1_h = vld1q_s16(r1 + i + 8);
+ const int16x8_t d_l = vld1q_s16(d + i);
+ const int16x8_t d_h = vld1q_s16(d + i + 8);
+ // The following three lines are a bit inelegant compared to using a pair
+ // of vmovl_u8()... but it forces the compiler to emit a ZIP1, ZIP2 pair -
+ // which can be executed in parallel with the subsequent SSHL instructions.
+ // (SSHL can only be executed on half of the Neon pipes in modern Arm
+ // cores, whereas ZIP1/2 can be executed on all of them.)
+ const uint8x16x2_t m_u16 = vzipq_u8(vld1q_u8(m + i), vdupq_n_u8(0));
+ const int16x8_t m_l = vreinterpretq_s16_u8(m_u16.val[0]);
+ const int16x8_t m_h = vreinterpretq_s16_u8(m_u16.val[1]);
+
+ sum[0] = vshll_n_s16(vget_low_s16(r1_l), WEDGE_WEIGHT_BITS);
+ sum[1] = vshll_n_s16(vget_high_s16(r1_l), WEDGE_WEIGHT_BITS);
+ sum[2] = vshll_n_s16(vget_low_s16(r1_h), WEDGE_WEIGHT_BITS);
+ sum[3] = vshll_n_s16(vget_high_s16(r1_h), WEDGE_WEIGHT_BITS);
+
+ sum[0] = vmlal_s16(sum[0], vget_low_s16(m_l), vget_low_s16(d_l));
+ sum[1] = vmlal_s16(sum[1], vget_high_s16(m_l), vget_high_s16(d_l));
+ sum[2] = vmlal_s16(sum[2], vget_low_s16(m_h), vget_low_s16(d_h));
+ sum[3] = vmlal_s16(sum[3], vget_high_s16(m_h), vget_high_s16(d_h));
+
+ sum_s16[0] = vqmovn_s32(sum[0]);
+ sum_s16[1] = vqmovn_s32(sum[1]);
+ sum_s16[2] = vqmovn_s32(sum[2]);
+ sum_s16[3] = vqmovn_s32(sum[3]);
+
+ sse[0] = vmull_s16(sum_s16[0], sum_s16[0]);
+ sse[1] = vmull_s16(sum_s16[2], sum_s16[2]);
+ sse[0] = vmlal_s16(sse[0], sum_s16[1], sum_s16[1]);
+ sse[1] = vmlal_s16(sse[1], sum_s16[3], sum_s16[3]);
+
+ v_csse[0] = vpadalq_u32(v_csse[0], vreinterpretq_u32_s32(sse[0]));
+ v_csse[1] = vpadalq_u32(v_csse[1], vreinterpretq_u32_s32(sse[1]));
+
+ i += 16;
+ } while (i < N);
+
+ uint64_t csse = horizontal_add_u64x2(vaddq_u64(v_csse[0], v_csse[1]));
+ return ROUND_POWER_OF_TWO(csse, 2 * WEDGE_WEIGHT_BITS);
+}
diff --git a/test/av1_wedge_utils_test.cc b/test/av1_wedge_utils_test.cc
index 369479a..46f6d92 100644
--- a/test/av1_wedge_utils_test.cc
+++ b/test/av1_wedge_utils_test.cc
@@ -374,6 +374,13 @@
av1_wedge_compute_delta_squares_sse2)));
#endif // HAVE_SSE2
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, WedgeUtilsSSEOptTest,
+ ::testing::Values(TestFuncsFSSE(av1_wedge_sse_from_residuals_c,
+ av1_wedge_sse_from_residuals_neon)));
+#endif // HAVE_NEON
+
#if HAVE_AVX2
INSTANTIATE_TEST_SUITE_P(
AVX2, WedgeUtilsSSEOptTest,
