Add SSSE3 four points search masked_sad
Change-Id: I36a556d1f517536e2d6bd65ccf0c94ca65b7fcad
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake
index 16ceaba..c019688 100644
--- a/aom_dsp/aom_dsp.cmake
+++ b/aom_dsp/aom_dsp.cmake
@@ -254,6 +254,7 @@
list(APPEND AOM_DSP_ENCODER_INTRIN_SSSE3
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.h"
"${AOM_ROOT}/aom_dsp/x86/masked_sad_intrin_ssse3.c"
+ "${AOM_ROOT}/aom_dsp/x86/masked_sad4d_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.h"
"${AOM_ROOT}/aom_dsp/x86/masked_variance_intrin_ssse3.c"
"${AOM_ROOT}/aom_dsp/x86/quantize_ssse3.c"
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index eabb9bf..904044d 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -824,6 +824,7 @@
($w, $h) = @$_;
add_proto qw/void/, "aom_sad${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
add_proto qw/void/, "aom_sad${w}x${h}x4d_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, const uint8_t *second_pred, uint32_t *sad_array";
+ add_proto qw/void/, "aom_masked_sad${w}x${h}x4d", "const uint8_t *src, int src_stride, const uint8_t *ref[], int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask, unsigned sads[]";
}
specialize qw/aom_sad128x128x4d avx2 sse2/;
@@ -883,6 +884,35 @@
specialize qw/aom_sad8x32x4d_avg sse2/;
specialize qw/aom_sad32x8x4d_avg sse2/;
specialize qw/aom_sad64x16x4d_avg sse2/;
+
+ specialize qw/aom_masked_sad128x128x4d ssse3/;
+ specialize qw/aom_masked_sad128x64x4d ssse3/;
+ specialize qw/aom_masked_sad64x128x4d ssse3/;
+ specialize qw/aom_masked_sad64x64x4d ssse3/;
+ specialize qw/aom_masked_sad64x32x4d ssse3/;
+ specialize qw/aom_masked_sad64x16x4d ssse3/;
+ specialize qw/aom_masked_sad32x64x4d ssse3/;
+ specialize qw/aom_masked_sad32x32x4d ssse3/;
+ specialize qw/aom_masked_sad32x16x4d ssse3/;
+ specialize qw/aom_masked_sad32x8x4d ssse3/;
+ specialize qw/aom_masked_sad16x64x4d ssse3/;
+ specialize qw/aom_masked_sad16x32x4d ssse3/;
+ specialize qw/aom_masked_sad16x16x4d ssse3/;
+ specialize qw/aom_masked_sad16x8x4d ssse3/;
+
+ specialize qw/aom_masked_sad8x16x4d ssse3/;
+ specialize qw/aom_masked_sad8x8x4d ssse3/;
+ specialize qw/aom_masked_sad8x4x4d ssse3/;
+ specialize qw/aom_masked_sad4x16x4d ssse3/;
+ specialize qw/aom_masked_sad4x8x4d ssse3/;
+ specialize qw/aom_masked_sad4x4x4d ssse3/;
+
+ specialize qw/aom_masked_sad4x32x4d ssse3/;
+ specialize qw/aom_masked_sad4x16x4d ssse3/;
+ specialize qw/aom_masked_sad16x4x4d ssse3/;
+ specialize qw/aom_masked_sad8x32x4d ssse3/;
+ specialize qw/aom_masked_sad32x8x4d ssse3/;
+ specialize qw/aom_masked_sad64x16x4d ssse3/;
#
# Multi-block SAD, comparing a reference to N independent blocks
#
diff --git a/aom_dsp/sad_av1.c b/aom_dsp/sad_av1.c
index 1ff3d4f..4675181 100644
--- a/aom_dsp/sad_av1.c
+++ b/aom_dsp/sad_av1.c
@@ -49,6 +49,21 @@
else \
return masked_sad(src, src_stride, second_pred, m, ref, ref_stride, msk, \
msk_stride, m, n); \
+ } \
+ void aom_masked_sad##m##x##n##x4d_c( \
+ const uint8_t *src, int src_stride, const uint8_t *ref[], \
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
+ int msk_stride, int invert_mask, unsigned sads[]) { \
+ if (!invert_mask) \
+ for (int i = 0; i < 4; i++) { \
+ sads[i] = masked_sad(src, src_stride, ref[i], ref_stride, second_pred, \
+ m, msk, msk_stride, m, n); \
+ } \
+ else \
+ for (int i = 0; i < 4; i++) { \
+ sads[i] = masked_sad(src, src_stride, second_pred, m, ref[i], \
+ ref_stride, msk, msk_stride, m, n); \
+ } \
}
/* clang-format off */
diff --git a/aom_dsp/x86/masked_sad4d_ssse3.c b/aom_dsp/x86/masked_sad4d_ssse3.c
new file mode 100644
index 0000000..8ef7ee0
--- /dev/null
+++ b/aom_dsp/x86/masked_sad4d_ssse3.c
@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <stdio.h>
+#include <tmmintrin.h>
+
+#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
+
+#include "aom_dsp/blend.h"
+#include "aom/aom_integer.h"
+#include "aom_dsp/x86/synonyms.h"
+
+#include "aom_dsp/x86/masked_sad_intrin_ssse3.h"
+
+#define MASK_SAD16XH_ONE_REF(idx) \
+ a = _mm_loadu_si128((const __m128i *)&ref##idx[x]); \
+ data_l = _mm_unpacklo_epi8(a, b); \
+ mask_l = _mm_unpacklo_epi8(m, m_inv); \
+ pred_l = _mm_maddubs_epi16(data_l, mask_l); \
+ pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS); \
+ \
+ data_r = _mm_unpackhi_epi8(a, b); \
+ mask_r = _mm_unpackhi_epi8(m, m_inv); \
+ pred_r = _mm_maddubs_epi16(data_r, mask_r); \
+ pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS); \
+ \
+ pred = _mm_packus_epi16(pred_l, pred_r); \
+ res##idx = _mm_add_epi32(res##idx, _mm_sad_epu8(pred, src));
+
+static INLINE void masked_sadx4d_ssse3(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *a_ptr[], int a_stride,
+ const uint8_t *b_ptr, int b_stride,
+ const uint8_t *m_ptr, int m_stride,
+ int width, int height, int inv_mask,
+ unsigned sad_array[]) {
+ int x, y;
+ __m128i a;
+ __m128i data_l, data_r, mask_l, mask_r, pred_l, pred_r, pred;
+ const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
+ __m128i res0 = _mm_setzero_si128();
+ __m128i res1 = _mm_setzero_si128();
+ __m128i res2 = _mm_setzero_si128();
+ __m128i res3 = _mm_setzero_si128();
+ const uint8_t *ref0 = a_ptr[0];
+ const uint8_t *ref1 = a_ptr[1];
+ const uint8_t *ref2 = a_ptr[2];
+ const uint8_t *ref3 = a_ptr[3];
+
+ for (y = 0; y < height; y++) {
+ for (x = 0; x < width; x += 16) {
+ const __m128i src = _mm_loadu_si128((const __m128i *)&src_ptr[x]);
+ const __m128i b = _mm_loadu_si128((const __m128i *)&b_ptr[x]);
+ const __m128i m_copy = _mm_loadu_si128((const __m128i *)&m_ptr[x]);
+ __m128i m_inv = _mm_sub_epi8(mask_max, m_copy);
+ __m128i m = inv_mask ? m_inv : m_copy;
+ m_inv = inv_mask ? m_copy : m_inv;
+
+ MASK_SAD16XH_ONE_REF(0)
+ MASK_SAD16XH_ONE_REF(1)
+ MASK_SAD16XH_ONE_REF(2)
+ MASK_SAD16XH_ONE_REF(3)
+ }
+
+ src_ptr += src_stride;
+ ref0 += a_stride;
+ ref1 += a_stride;
+ ref2 += a_stride;
+ ref3 += a_stride;
+ b_ptr += b_stride;
+ m_ptr += m_stride;
+ }
+ res0 = _mm_add_epi32(_mm_unpacklo_epi32(res0, res1),
+ _mm_unpackhi_epi32(res0, res1));
+ res2 = _mm_add_epi32(_mm_unpacklo_epi32(res2, res3),
+ _mm_unpackhi_epi32(res2, res3));
+
+ res0 = _mm_unpacklo_epi64(res0, res2);
+ _mm_storeu_si128((__m128i *)sad_array, res0);
+}
+
+#define MASK_SAD8XH_ONE_REF(idx) \
+ const __m128i a##idx##0 = _mm_loadl_epi64((__m128i *)ref##idx); \
+ const __m128i a##idx##1 = _mm_loadl_epi64((__m128i *)(ref##idx + a_stride)); \
+ data_l = _mm_unpacklo_epi8(a##idx##0, b0); \
+ mask_l = _mm_unpacklo_epi8(m, m_inv); \
+ pred_l = _mm_maddubs_epi16(data_l, mask_l); \
+ pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS); \
+ \
+ data_r = _mm_unpacklo_epi8(a##idx##1, b1); \
+ mask_r = _mm_unpackhi_epi8(m, m_inv); \
+ pred_r = _mm_maddubs_epi16(data_r, mask_r); \
+ pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS); \
+ \
+ pred = _mm_packus_epi16(pred_l, pred_r); \
+ res##idx = _mm_add_epi32(res##idx, _mm_sad_epu8(pred, src));
+
+void aom_masked_sad8xhx4d_ssse3(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_array[], int a_stride,
+ const uint8_t *b_ptr, int b_stride,
+ const uint8_t *m_ptr, int m_stride, int height,
+ int inv_mask, unsigned sad_array[]) {
+ const uint8_t *ref0 = ref_array[0];
+ const uint8_t *ref1 = ref_array[1];
+ const uint8_t *ref2 = ref_array[2];
+ const uint8_t *ref3 = ref_array[3];
+ __m128i data_l, data_r, pred_l, pred_r, mask_l, mask_r, pred;
+ __m128i res0 = _mm_setzero_si128();
+ __m128i res1 = _mm_setzero_si128();
+ __m128i res2 = _mm_setzero_si128();
+ __m128i res3 = _mm_setzero_si128();
+ const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
+
+ for (int y = 0; y < height; y += 2) {
+ const __m128i src = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)src_ptr),
+ _mm_loadl_epi64((const __m128i *)(src_ptr + src_stride)));
+ const __m128i b0 = _mm_loadl_epi64((__m128i *)b_ptr);
+ const __m128i b1 = _mm_loadl_epi64((__m128i *)(b_ptr + b_stride));
+ const __m128i m0 = _mm_loadl_epi64((__m128i *)m_ptr);
+ const __m128i m1 = _mm_loadl_epi64((__m128i *)(m_ptr + m_stride));
+ __m128i m_copy = _mm_unpacklo_epi64(m0, m1);
+ __m128i m_inv = _mm_sub_epi8(mask_max, m_copy);
+ __m128i m = inv_mask ? m_inv : m_copy;
+ m_inv = inv_mask ? m_copy : m_inv;
+
+ MASK_SAD8XH_ONE_REF(0)
+ MASK_SAD8XH_ONE_REF(1)
+ MASK_SAD8XH_ONE_REF(2)
+ MASK_SAD8XH_ONE_REF(3)
+
+ ref0 += 2 * a_stride;
+ ref1 += 2 * a_stride;
+ ref2 += 2 * a_stride;
+ ref3 += 2 * a_stride;
+ src_ptr += 2 * src_stride;
+ b_ptr += 2 * b_stride;
+ m_ptr += 2 * m_stride;
+ }
+ res0 = _mm_add_epi32(_mm_unpacklo_epi32(res0, res1),
+ _mm_unpackhi_epi32(res0, res1));
+ res2 = _mm_add_epi32(_mm_unpacklo_epi32(res2, res3),
+ _mm_unpackhi_epi32(res2, res3));
+ res0 = _mm_unpacklo_epi64(res0, res2);
+ _mm_storeu_si128((__m128i *)sad_array, res0);
+}
+
+#define MASK_SAD4XH_ONE_REF(idx) \
+ a = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(uint32_t *)ref##idx), \
+ _mm_cvtsi32_si128(*(uint32_t *)&ref##idx[a_stride])); \
+ data = _mm_unpacklo_epi8(a, b); \
+ mask = _mm_unpacklo_epi8(m, m_inv); \
+ pred = _mm_maddubs_epi16(data, mask); \
+ pred = xx_roundn_epu16(pred, AOM_BLEND_A64_ROUND_BITS); \
+ \
+ pred = _mm_packus_epi16(pred, _mm_setzero_si128()); \
+ res##idx = _mm_add_epi32(res##idx, _mm_sad_epu8(pred, src));
+
+void aom_masked_sad4xhx4d_ssse3(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_array[], int a_stride,
+ const uint8_t *b_ptr, int b_stride,
+ const uint8_t *m_ptr, int m_stride, int height,
+ int inv_mask, unsigned sad_array[]) {
+ const uint8_t *ref0 = ref_array[0];
+ const uint8_t *ref1 = ref_array[1];
+ const uint8_t *ref2 = ref_array[2];
+ const uint8_t *ref3 = ref_array[3];
+ __m128i data, pred, mask;
+ __m128i res0 = _mm_setzero_si128();
+ __m128i res1 = _mm_setzero_si128();
+ __m128i res2 = _mm_setzero_si128();
+ __m128i res3 = _mm_setzero_si128();
+ __m128i a;
+ const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
+
+ for (int y = 0; y < height; y += 2) {
+ const __m128i src = _mm_unpacklo_epi32(
+ _mm_cvtsi32_si128(*(uint32_t *)src_ptr),
+ _mm_cvtsi32_si128(*(uint32_t *)&src_ptr[src_stride]));
+ const __m128i b =
+ _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(uint32_t *)b_ptr),
+ _mm_cvtsi32_si128(*(uint32_t *)&b_ptr[b_stride]));
+ const __m128i m_copy =
+ _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(uint32_t *)m_ptr),
+ _mm_cvtsi32_si128(*(uint32_t *)&m_ptr[m_stride]));
+
+ __m128i m_inv = _mm_sub_epi8(mask_max, m_copy);
+ __m128i m = inv_mask ? m_inv : m_copy;
+ m_inv = inv_mask ? m_copy : m_inv;
+
+ MASK_SAD4XH_ONE_REF(0)
+ MASK_SAD4XH_ONE_REF(1)
+ MASK_SAD4XH_ONE_REF(2)
+ MASK_SAD4XH_ONE_REF(3)
+
+ ref0 += 2 * a_stride;
+ ref1 += 2 * a_stride;
+ ref2 += 2 * a_stride;
+ ref3 += 2 * a_stride;
+ src_ptr += 2 * src_stride;
+ b_ptr += 2 * b_stride;
+ m_ptr += 2 * m_stride;
+ }
+ res0 = _mm_unpacklo_epi32(res0, res1);
+ res2 = _mm_unpacklo_epi32(res2, res3);
+ res0 = _mm_unpacklo_epi64(res0, res2);
+ _mm_storeu_si128((__m128i *)sad_array, res0);
+}
+
+#define MASKSADMXN_SSSE3(m, n) \
+ void aom_masked_sad##m##x##n##x4d_ssse3( \
+ const uint8_t *src, int src_stride, const uint8_t *ref[], \
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
+ int msk_stride, int inv_mask, unsigned sad_array[]) { \
+ masked_sadx4d_ssse3(src, src_stride, ref, ref_stride, second_pred, m, msk, \
+ msk_stride, m, n, inv_mask, sad_array); \
+ }
+
+#define MASKSAD8XN_SSSE3(n) \
+ void aom_masked_sad8x##n##x4d_ssse3( \
+ const uint8_t *src, int src_stride, const uint8_t *ref[], \
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
+ int msk_stride, int inv_mask, unsigned sad_array[]) { \
+ aom_masked_sad8xhx4d_ssse3(src, src_stride, ref, ref_stride, second_pred, \
+ 8, msk, msk_stride, n, inv_mask, sad_array); \
+ }
+
+#define MASKSAD4XN_SSSE3(n) \
+ void aom_masked_sad4x##n##x4d_ssse3( \
+ const uint8_t *src, int src_stride, const uint8_t *ref[], \
+ int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
+ int msk_stride, int inv_mask, unsigned sad_array[]) { \
+ aom_masked_sad4xhx4d_ssse3(src, src_stride, ref, ref_stride, second_pred, \
+ 4, msk, msk_stride, n, inv_mask, sad_array); \
+ }
+
+MASKSADMXN_SSSE3(128, 128)
+MASKSADMXN_SSSE3(128, 64)
+MASKSADMXN_SSSE3(64, 128)
+MASKSADMXN_SSSE3(64, 64)
+MASKSADMXN_SSSE3(64, 32)
+MASKSADMXN_SSSE3(32, 64)
+MASKSADMXN_SSSE3(32, 32)
+MASKSADMXN_SSSE3(32, 16)
+MASKSADMXN_SSSE3(16, 32)
+MASKSADMXN_SSSE3(16, 16)
+MASKSADMXN_SSSE3(16, 8)
+MASKSAD8XN_SSSE3(16)
+MASKSAD8XN_SSSE3(8)
+MASKSAD8XN_SSSE3(4)
+MASKSAD4XN_SSSE3(8)
+MASKSAD4XN_SSSE3(4)
+MASKSAD4XN_SSSE3(16)
+MASKSADMXN_SSSE3(16, 4)
+MASKSAD8XN_SSSE3(32)
+MASKSADMXN_SSSE3(32, 8)
+MASKSADMXN_SSSE3(16, 64)
+MASKSADMXN_SSSE3(64, 16)
diff --git a/test/masked_sad_test.cc b/test/masked_sad_test.cc
index 757369a..a281739 100644
--- a/test/masked_sad_test.cc
+++ b/test/masked_sad_test.cc
@@ -36,7 +36,35 @@
int invert_mask);
typedef std::tuple<MaskedSADFunc, MaskedSADFunc> MaskedSADParam;
-class MaskedSADTest : public ::testing::TestWithParam<MaskedSADParam> {
+typedef void (*MaskedSADx4Func)(const uint8_t *src, int src_stride,
+ const uint8_t *ref[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int invert_mask,
+ unsigned sads[]);
+
+typedef std::tuple<MaskedSADx4Func, MaskedSADx4Func> MaskedSADx4Param;
+
+class MaskedSADTestBase : public ::testing::Test {
+ public:
+ virtual ~MaskedSADTestBase() {}
+ virtual void SetUp() = 0;
+ virtual void runRef(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times) = 0;
+ virtual void runTest(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times) = 0;
+
+ virtual void TearDown() { libaom_test::ClearSystemState(); }
+ void runMaskedSADTest(int run_times);
+};
+
+class MaskedSADTest : public MaskedSADTestBase,
+ public ::testing::WithParamInterface<MaskedSADParam> {
public:
virtual ~MaskedSADTest() {}
virtual void SetUp() {
@@ -44,20 +72,113 @@
ref_maskedSAD_op_ = GET_PARAM(1);
}
- virtual void TearDown() { libaom_test::ClearSystemState(); }
- void runMaskedSADTest(int run_times);
+ virtual void runRef(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[], int times);
+ virtual void runTest(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times);
protected:
MaskedSADFunc maskedSAD_op_;
MaskedSADFunc ref_maskedSAD_op_;
};
-void MaskedSADTest::runMaskedSADTest(int run_times) {
- unsigned int ref_ret = 0, ret = 1;
+
+class MaskedSADx4Test : public MaskedSADTestBase,
+ public ::testing::WithParamInterface<MaskedSADx4Param> {
+ public:
+ virtual ~MaskedSADx4Test() {}
+ virtual void SetUp() {
+ maskedSAD_op_ = GET_PARAM(0);
+ ref_maskedSAD_op_ = GET_PARAM(1);
+ }
+ virtual void runRef(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[], int times);
+ virtual void runTest(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int inv_mask, unsigned sads[],
+ int times);
+
+ protected:
+ MaskedSADx4Func maskedSAD_op_;
+ MaskedSADx4Func ref_maskedSAD_op_;
+};
+
+void MaskedSADTest::runRef(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int invert_mask, unsigned sads[],
+ int times) {
+ for (int repeat = 0; repeat < times; ++repeat) {
+ sads[0] = ref_maskedSAD_op_(src_ptr, src_stride, ref_ptr[0], ref_stride,
+ second_pred, msk, msk_stride, invert_mask);
+ }
+}
+
+void MaskedSADTest::runTest(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int invert_mask, unsigned sads[],
+ int times) {
+ if (times == 1) {
+ sads[0] = maskedSAD_op_(src_ptr, src_stride, ref_ptr[0], ref_stride,
+ second_pred, msk, msk_stride, invert_mask);
+ } else {
+ for (int repeat = 0; repeat < times; ++repeat) {
+ ASM_REGISTER_STATE_CHECK(
+ sads[0] = maskedSAD_op_(src_ptr, src_stride, ref_ptr[0], ref_stride,
+ second_pred, msk, msk_stride, invert_mask));
+ }
+ }
+}
+
+void MaskedSADx4Test::runRef(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int invert_mask, unsigned sads[],
+ int times) {
+ for (int repeat = 0; repeat < times; ++repeat) {
+ ref_maskedSAD_op_(src_ptr, src_stride, ref_ptr, ref_stride, second_pred,
+ msk, msk_stride, invert_mask, sads);
+ }
+}
+
+void MaskedSADx4Test::runTest(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr[], int ref_stride,
+ const uint8_t *second_pred, const uint8_t *msk,
+ int msk_stride, int invert_mask, unsigned sads[],
+ int times) {
+ if (times == 1) {
+ ASM_REGISTER_STATE_CHECK(maskedSAD_op_(src_ptr, src_stride, ref_ptr,
+ ref_stride, second_pred, msk,
+ msk_stride, invert_mask, sads));
+ } else {
+ for (int repeat = 0; repeat < times; ++repeat) {
+ maskedSAD_op_(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, msk,
+ msk_stride, invert_mask, sads);
+ }
+ }
+}
+
+void MaskedSADTestBase::runMaskedSADTest(int run_times) {
ACMRandom rnd(ACMRandom::DeterministicSeed());
+ const unsigned kBlockSize = MAX_SB_SIZE * MAX_SB_SIZE;
DECLARE_ALIGNED(16, uint8_t, src_ptr[MAX_SB_SIZE * MAX_SB_SIZE]);
- DECLARE_ALIGNED(16, uint8_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE]);
+ DECLARE_ALIGNED(16, uint8_t, ref_ptr[MAX_SB_SIZE * MAX_SB_SIZE * 4]);
DECLARE_ALIGNED(16, uint8_t, second_pred_ptr[MAX_SB_SIZE * MAX_SB_SIZE]);
DECLARE_ALIGNED(16, uint8_t, msk_ptr[MAX_SB_SIZE * MAX_SB_SIZE]);
+
+ const uint8_t *refs[] = { ref_ptr, ref_ptr + kBlockSize,
+ ref_ptr + 2 * kBlockSize,
+ ref_ptr + 3 * kBlockSize };
+ unsigned sads[] = { 0, 0, 0, 0 };
+ unsigned sads_ref[] = { 0, 0, 0, 0 };
int err_count = 0;
int first_failure = -1;
int src_stride = MAX_SB_SIZE;
@@ -76,33 +197,23 @@
for (int invert_mask = 0; invert_mask < 2; ++invert_mask) {
aom_usec_timer timer;
aom_usec_timer_start(&timer);
- for (int repeat = 0; repeat < run_times; ++repeat) {
- ref_ret = ref_maskedSAD_op_(src_ptr, src_stride, ref_ptr, ref_stride,
- second_pred_ptr, msk_ptr, msk_stride,
- invert_mask);
- }
+ runRef(src_ptr, src_stride, refs, ref_stride, second_pred_ptr, msk_ptr,
+ msk_stride, invert_mask, sads_ref, run_times);
aom_usec_timer_mark(&timer);
const double time1 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
aom_usec_timer_start(&timer);
- if (run_times == 1) {
- ASM_REGISTER_STATE_CHECK(ret = maskedSAD_op_(src_ptr, src_stride,
- ref_ptr, ref_stride,
- second_pred_ptr, msk_ptr,
- msk_stride, invert_mask));
- } else {
- for (int repeat = 0; repeat < run_times; ++repeat) {
- ret =
- maskedSAD_op_(src_ptr, src_stride, ref_ptr, ref_stride,
- second_pred_ptr, msk_ptr, msk_stride, invert_mask);
- }
- }
+ runTest(src_ptr, src_stride, refs, ref_stride, second_pred_ptr, msk_ptr,
+ msk_stride, invert_mask, sads, run_times);
aom_usec_timer_mark(&timer);
const double time2 = static_cast<double>(aom_usec_timer_elapsed(&timer));
+
if (run_times > 10) {
printf("%7.2f/%7.2fns", time1, time2);
printf("(%3.2f)\n", time1 / time2);
}
- if (ret != ref_ret) {
+ if (sads_ref[0] != sads[0] || sads_ref[1] != sads[1] ||
+ sads_ref[2] != sads[2] || sads_ref[3] != sads[3]) {
err_count++;
if (first_failure == -1) first_failure = i;
}
@@ -116,6 +227,10 @@
TEST_P(MaskedSADTest, DISABLED_Speed) { runMaskedSADTest(2000000); }
+TEST_P(MaskedSADx4Test, OperationCheck) { runMaskedSADTest(1); }
+
+TEST_P(MaskedSADx4Test, DISABLED_Speed) { runMaskedSADTest(2000000); }
+
#if CONFIG_AV1_HIGHBITDEPTH
typedef unsigned int (*HighbdMaskedSADFunc)(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
@@ -240,6 +355,34 @@
INSTANTIATE_TEST_SUITE_P(SSSE3, MaskedSADTest, ::testing::ValuesIn(msad_test));
+const MaskedSADx4Param msadx4_test[] = {
+ make_tuple(&aom_masked_sad4x4x4d_ssse3, &aom_masked_sad4x4x4d_c),
+ make_tuple(&aom_masked_sad4x8x4d_ssse3, &aom_masked_sad4x8x4d_c),
+ make_tuple(&aom_masked_sad8x4x4d_ssse3, &aom_masked_sad8x4x4d_c),
+ make_tuple(&aom_masked_sad8x8x4d_ssse3, &aom_masked_sad8x8x4d_c),
+ make_tuple(&aom_masked_sad8x16x4d_ssse3, &aom_masked_sad8x16x4d_c),
+ make_tuple(&aom_masked_sad16x8x4d_ssse3, &aom_masked_sad16x8x4d_c),
+ make_tuple(&aom_masked_sad16x16x4d_ssse3, &aom_masked_sad16x16x4d_c),
+ make_tuple(&aom_masked_sad16x32x4d_ssse3, &aom_masked_sad16x32x4d_c),
+ make_tuple(&aom_masked_sad32x16x4d_ssse3, &aom_masked_sad32x16x4d_c),
+ make_tuple(&aom_masked_sad32x32x4d_ssse3, &aom_masked_sad32x32x4d_c),
+ make_tuple(&aom_masked_sad32x64x4d_ssse3, &aom_masked_sad32x64x4d_c),
+ make_tuple(&aom_masked_sad64x32x4d_ssse3, &aom_masked_sad64x32x4d_c),
+ make_tuple(&aom_masked_sad64x64x4d_ssse3, &aom_masked_sad64x64x4d_c),
+ make_tuple(&aom_masked_sad64x128x4d_ssse3, &aom_masked_sad64x128x4d_c),
+ make_tuple(&aom_masked_sad128x64x4d_ssse3, &aom_masked_sad128x64x4d_c),
+ make_tuple(&aom_masked_sad128x128x4d_ssse3, &aom_masked_sad128x128x4d_c),
+ make_tuple(&aom_masked_sad4x16x4d_ssse3, &aom_masked_sad4x16x4d_c),
+ make_tuple(&aom_masked_sad16x4x4d_ssse3, &aom_masked_sad16x4x4d_c),
+ make_tuple(&aom_masked_sad8x32x4d_ssse3, &aom_masked_sad8x32x4d_c),
+ make_tuple(&aom_masked_sad32x8x4d_ssse3, &aom_masked_sad32x8x4d_c),
+ make_tuple(&aom_masked_sad16x64x4d_ssse3, &aom_masked_sad16x64x4d_c),
+ make_tuple(&aom_masked_sad64x16x4d_ssse3, &aom_masked_sad64x16x4d_c),
+};
+
+INSTANTIATE_TEST_SUITE_P(SSSE3, MaskedSADx4Test,
+ ::testing::ValuesIn(msadx4_test));
+
#if CONFIG_AV1_HIGHBITDEPTH
const HighbdMaskedSADParam hbd_msad_test[] = {
make_tuple(&aom_highbd_masked_sad4x4_ssse3, &aom_highbd_masked_sad4x4_c),
