Add NEON version of av1_lowbd_pixel_proj_error
SpeedUp
r0 0 r1 0 gain (7.62)
r0 1 r1 0 gain (4.03)
r0 0 r1 1 gain (4.03)
r0 1 r1 1 gain (3.73)
via NEON/PixelProjErrorTest.DISABLED_Speed
Change-Id: I317ec9e8725f1a088aa33682d0099f0b052918f5
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 0099a21..94d2102 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -376,6 +376,7 @@
list(APPEND AOM_AV1_ENCODER_INTRIN_NEON
"${AOM_ROOT}/av1/encoder/arm/neon/quantize_neon.c"
+ "${AOM_ROOT}/av1/encoder/arm/neon/picksrt_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/av1_error_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/encodetxb_neon.c"
"${AOM_ROOT}/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c"
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index a9b2942..3869447 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -396,7 +396,7 @@
specialize qw/av1_calc_proj_params avx2/;
add_proto qw/int64_t av1_lowbd_pixel_proj_error/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params";
- specialize qw/av1_lowbd_pixel_proj_error sse4_1 avx2/;
+ specialize qw/av1_lowbd_pixel_proj_error sse4_1 avx2 neon/;
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/int64_t av1_highbd_pixel_proj_error/, " const uint8_t *src8, int width, int height, int src_stride, const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride, int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params";
diff --git a/av1/encoder/arm/neon/picksrt_neon.c b/av1/encoder/arm/neon/picksrt_neon.c
new file mode 100644
index 0000000..34595e8
--- /dev/null
+++ b/av1/encoder/arm/neon/picksrt_neon.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <arm_neon.h>
+#include <math.h>
+
+#include "aom/aom_integer.h"
+#include "aom_mem/aom_mem.h"
+#include "aom_ports/mem.h"
+#include "av1/common/restoration.h"
+#include "common/tools_common.h"
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+int64_t av1_lowbd_pixel_proj_error_neon(
+ const uint8_t *src8, int width, int height, int src_stride,
+ const uint8_t *dat8, int dat_stride, int32_t *flt0, int flt0_stride,
+ int32_t *flt1, int flt1_stride, int xq[2], const sgr_params_type *params) {
+ int i, j, k;
+ const int32_t shift = SGRPROJ_RST_BITS + SGRPROJ_PRJ_BITS;
+ const int32x4_t zero = vdupq_n_s32(0);
+ uint64x2_t sum64 = vreinterpretq_u64_s32(zero);
+ const uint8_t *src = src8;
+ const uint8_t *dat = dat8;
+
+ int64_t err = 0;
+ if (params->r[0] > 0 && params->r[1] > 0) {
+ for (i = 0; i < height; ++i) {
+ int32x4_t err0 = zero;
+ for (j = 0; j <= width - 8; j += 8) {
+ const uint8x8_t d0 = vld1_u8(&dat[j]);
+ const uint8x8_t s0 = vld1_u8(&src[j]);
+ const int16x8_t flt0_16b =
+ vcombine_s16(vqmovn_s32(vld1q_s32(&flt0[j])),
+ vqmovn_s32(vld1q_s32(&flt0[j + 4])));
+ const int16x8_t flt1_16b =
+ vcombine_s16(vqmovn_s32(vld1q_s32(&flt1[j])),
+ vqmovn_s32(vld1q_s32(&flt1[j + 4])));
+ const int16x8_t u0 =
+ vreinterpretq_s16_u16(vshll_n_u8(d0, SGRPROJ_RST_BITS));
+ const int16x8_t flt0_0_sub_u = vsubq_s16(flt0_16b, u0);
+ const int16x8_t flt1_0_sub_u = vsubq_s16(flt1_16b, u0);
+ const int16x4_t flt0_16b_sub_u_lo = vget_low_s16(flt0_0_sub_u);
+ const int16x4_t flt0_16b_sub_u_hi = vget_high_s16(flt0_0_sub_u);
+ const int16x4_t flt1_16b_sub_u_lo = vget_low_s16(flt1_0_sub_u);
+ const int16x4_t flt1_16b_sub_u_hi = vget_high_s16(flt1_0_sub_u);
+
+ int32x4_t v0 = vmull_n_s16(flt0_16b_sub_u_lo, (int16_t)xq[0]);
+ v0 = vmlal_n_s16(v0, flt1_16b_sub_u_lo, (int16_t)xq[1]);
+ int32x4_t v1 = vmull_n_s16(flt0_16b_sub_u_hi, (int16_t)xq[0]);
+ v1 = vmlal_n_s16(v1, flt1_16b_sub_u_hi, (int16_t)xq[1]);
+ const int16x4_t vr0 = vqrshrn_n_s32(v0, 11);
+ const int16x4_t vr1 = vqrshrn_n_s32(v1, 11);
+ const int16x8_t e0 = vaddq_s16(vcombine_s16(vr0, vr1),
+ vreinterpretq_s16_u16(vsubl_u8(d0, s0)));
+ const int16x4_t e0_lo = vget_low_s16(e0);
+ const int16x4_t e0_hi = vget_high_s16(e0);
+ err0 = vmlal_s16(err0, e0_lo, e0_lo);
+ err0 = vmlal_s16(err0, e0_hi, e0_hi);
+ }
+ for (k = j; k < width; ++k) {
+ const int32_t u = dat[k] << SGRPROJ_RST_BITS;
+ int32_t v = xq[0] * (flt0[k] - u) + xq[1] * (flt1[k] - u);
+ const int32_t e = ROUND_POWER_OF_TWO(v, 11) + dat[k] - src[k];
+ err += e * e;
+ }
+ dat += dat_stride;
+ src += src_stride;
+ flt0 += flt0_stride;
+ flt1 += flt1_stride;
+ sum64 = vpadalq_u32(sum64, vreinterpretq_u32_s32(err0));
+ }
+
+ } else if (params->r[0] > 0 || params->r[1] > 0) {
+ const int xq_active = (params->r[0] > 0) ? xq[0] : xq[1];
+ const int32_t *flt = (params->r[0] > 0) ? flt0 : flt1;
+ const int flt_stride = (params->r[0] > 0) ? flt0_stride : flt1_stride;
+ for (i = 0; i < height; ++i) {
+ int32x4_t err0 = zero;
+ for (j = 0; j <= width - 8; j += 8) {
+ const uint8x8_t d0 = vld1_u8(&dat[j]);
+ const uint8x8_t s0 = vld1_u8(&src[j]);
+ const uint16x8_t d0s0 = vsubl_u8(d0, s0);
+ const uint16x8x2_t d0w =
+ vzipq_u16(vmovl_u8(d0), vreinterpretq_u16_s32(zero));
+
+ const int32x4_t flt_16b_lo = vld1q_s32(&flt[j]);
+ const int32x4_t flt_16b_hi = vld1q_s32(&flt[j + 4]);
+
+ int32x4_t v0 = vmulq_n_s32(flt_16b_lo, xq_active);
+ v0 = vmlsq_n_s32(v0, vreinterpretq_s32_u16(d0w.val[0]),
+ xq_active << SGRPROJ_RST_BITS);
+ int32x4_t v1 = vmulq_n_s32(flt_16b_hi, xq_active);
+ v1 = vmlsq_n_s32(v1, vreinterpretq_s32_u16(d0w.val[1]),
+ xq_active << SGRPROJ_RST_BITS);
+ const int16x4_t vr0 = vqrshrn_n_s32(v0, 11);
+ const int16x4_t vr1 = vqrshrn_n_s32(v1, 11);
+ const int16x8_t e0 =
+ vaddq_s16(vcombine_s16(vr0, vr1), vreinterpretq_s16_u16(d0s0));
+ const int16x4_t e0_lo = vget_low_s16(e0);
+ const int16x4_t e0_hi = vget_high_s16(e0);
+ err0 = vmlal_s16(err0, e0_lo, e0_lo);
+ err0 = vmlal_s16(err0, e0_hi, e0_hi);
+ }
+ for (k = j; k < width; ++k) {
+ const int32_t u = dat[k] << SGRPROJ_RST_BITS;
+ int32_t v = xq_active * (flt[k] - u);
+ const int32_t e = ROUND_POWER_OF_TWO(v, shift) + dat[k] - src[k];
+ err += e * e;
+ }
+ dat += dat_stride;
+ src += src_stride;
+ flt += flt_stride;
+ sum64 = vpadalq_u32(sum64, vreinterpretq_u32_s32(err0));
+ }
+ } else {
+ uint32x4_t err0 = vreinterpretq_u32_s32(zero);
+ for (i = 0; i < height; ++i) {
+ for (j = 0; j <= width - 16; j += 16) {
+ const uint8x16_t d = vld1q_u8(&dat[j]);
+ const uint8x16_t s = vld1q_u8(&src[j]);
+ const uint8x16_t diff = vabdq_u8(d, s);
+ const uint8x8_t diff0 = vget_low_u8(diff);
+ const uint8x8_t diff1 = vget_high_u8(diff);
+ err0 = vpadalq_u16(err0, vmull_u8(diff0, diff0));
+ err0 = vpadalq_u16(err0, vmull_u8(diff1, diff1));
+ }
+ for (k = j; k < width; ++k) {
+ const int32_t e = dat[k] - src[k];
+ err += e * e;
+ }
+ dat += dat_stride;
+ src += src_stride;
+ }
+ sum64 = vpaddlq_u32(err0);
+ }
+#if defined(__aarch64__)
+ err += vaddvq_u64(sum64);
+#else
+ err += vget_lane_u64(vadd_u64(vget_low_u64(sum64), vget_high_u64(sum64)), 0);
+#endif // __aarch64__
+ return err;
+}
diff --git a/test/pickrst_test.cc b/test/pickrst_test.cc
index 9a2c5bc..77c6b47 100644
--- a/test/pickrst_test.cc
+++ b/test/pickrst_test.cc
@@ -188,6 +188,12 @@
::testing::Values(av1_lowbd_pixel_proj_error_avx2));
#endif // HAVE_AVX2
+#if HAVE_NEON
+
+INSTANTIATE_TEST_SUITE_P(NEON, PixelProjErrorTest,
+ ::testing::Values(av1_lowbd_pixel_proj_error_neon));
+#endif // HAVE_NEON
+
} // namespace pickrst_test_lowbd
#if CONFIG_AV1_HIGHBITDEPTH
