Implement Neon variance functions using UDOT instruction
Accelerate the Neon variance helper functions by implementing the sum
of squares calculation using a single Armv8.4-A UDOT instruction
instead of four MLAs.
The previous implementation is retained for use on systems that do
support the Armv8.4-A dot-product instructions.
Change-Id: I21689fd084c5ed9448fb1a630dc880ed45fac1df
diff --git a/aom_dsp/arm/variance_neon.c b/aom_dsp/arm/variance_neon.c
index 8505d1b..e8e3a53 100644
--- a/aom_dsp/arm/variance_neon.c
+++ b/aom_dsp/arm/variance_neon.c
@@ -18,6 +18,149 @@
#include "aom/aom_integer.h"
#include "aom_ports/mem.h"
+#if defined(__ARM_FEATURE_DOTPROD)
+
+static INLINE void variance_4xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = 0;
+ do {
+ uint8x16_t s = load_unaligned_u8q(src, src_stride);
+ uint8x16_t r = load_unaligned_u8q(ref, ref_stride);
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += 4 * src_stride;
+ ref += 4 * ref_stride;
+ i += 4;
+ } while (i < h);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_8xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = 0;
+ do {
+ uint8x16_t s = vcombine_u8(vld1_u8(src), vld1_u8(src + src_stride));
+ uint8x16_t r = vcombine_u8(vld1_u8(ref), vld1_u8(ref + ref_stride));
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += 2 * src_stride;
+ ref += 2 * ref_stride;
+ i += 2;
+ } while (i < h);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_16xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = 0;
+ do {
+ uint8x16_t s = vld1q_u8(src);
+ uint8x16_t r = vld1q_u8(ref);
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ src += src_stride;
+ ref += ref_stride;
+ i++;
+ } while (i < h);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_large_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int w, int h, uint32_t *sse, int *sum) {
+ uint32x4_t src_sum = vdupq_n_u32(0);
+ uint32x4_t ref_sum = vdupq_n_u32(0);
+ uint32x4_t sse_u32 = vdupq_n_u32(0);
+
+ int i = 0;
+ do {
+ int j = 0;
+ do {
+ uint8x16_t s = vld1q_u8(src + j);
+ uint8x16_t r = vld1q_u8(ref + j);
+
+ src_sum = vdotq_u32(src_sum, s, vdupq_n_u8(1));
+ ref_sum = vdotq_u32(ref_sum, r, vdupq_n_u8(1));
+
+ uint8x16_t abs_diff = vabdq_u8(s, r);
+ sse_u32 = vdotq_u32(sse_u32, abs_diff, abs_diff);
+
+ j += 16;
+ } while (j < w);
+
+ src += src_stride;
+ ref += ref_stride;
+ i++;
+ } while (i < h);
+
+ int32x4_t sum_diff =
+ vsubq_s32(vreinterpretq_s32_u32(src_sum), vreinterpretq_s32_u32(ref_sum));
+ *sum = horizontal_add_s32x4(sum_diff);
+ *sse = horizontal_add_u32x4(sse_u32);
+}
+
+static INLINE void variance_32xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ variance_large_neon(src, src_stride, ref, ref_stride, 32, h, sse, sum);
+}
+
+static INLINE void variance_64xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride, int h,
+ uint32_t *sse, int *sum) {
+ variance_large_neon(src, src_stride, ref, ref_stride, 64, h, sse, sum);
+}
+
+static INLINE void variance_128xh_neon(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ int h, uint32_t *sse, int *sum) {
+ variance_large_neon(src, src_stride, ref, ref_stride, 128, h, sse, sum);
+}
+
+#else // !defined(__ARM_FEATURE_DOTPROD)
+
static INLINE void variance_4xh_neon(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride, int h,
uint32_t *sse, int *sum) {
@@ -194,6 +337,8 @@
variance_large_neon(src, src_stride, ref, ref_stride, 128, h, 16, sse, sum);
}
+#endif // defined(__ARM_FEATURE_DOTPROD)
+
#define VARIANCE_WXH_NEON(w, h, shift) \
unsigned int aom_variance##w##x##h##_neon( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
