Add AVX2 optimization for cross_correlation module
Added compute_cross_correlation avx2 function and changed
test_bench to support AVX2 version.
Module level gains improved by a factor of ~1.45
on average w.r.t to SSE4_1 modules.
When tested for multiple test cases observed 0.4%
average reduction in encoder time for speed = 1 preset.
Change-Id: I7ee5d26d15b3d0bd36f5def64d0af58079ec604a
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 8c92615..19af0ff 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -282,6 +282,7 @@
list(APPEND AOM_AV1_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/av1/encoder/x86/av1_quantize_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_avx2.c"
+ "${AOM_ROOT}/av1/encoder/x86/corner_match_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm_avx2.h"
"${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm2d_avx2.c"
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index aa7fddf..e0ca13f 100755
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -336,7 +336,7 @@
if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/double compute_cross_correlation/, "unsigned char *im1, int stride1, int x1, int y1, unsigned char *im2, int stride2, int x2, int y2";
- specialize qw/compute_cross_correlation sse4_1/;
+ specialize qw/compute_cross_correlation sse4_1 avx2/;
}
# LOOP_RESTORATION functions
diff --git a/av1/encoder/x86/corner_match_avx2.c b/av1/encoder/x86/corner_match_avx2.c
new file mode 100644
index 0000000..3351d0a
--- /dev/null
+++ b/av1/encoder/x86/corner_match_avx2.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2018, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <stdlib.h>
+#include <memory.h>
+#include <math.h>
+#include <assert.h>
+
+#include <immintrin.h>
+#include "config/av1_rtcd.h"
+
+#include "aom_ports/mem.h"
+#include "av1/encoder/corner_match.h"
+
+DECLARE_ALIGNED(16, static const uint8_t, byte_mask[16]) = {
+ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0
+};
+#if MATCH_SZ != 13
+#error "Need to change byte_mask in corner_match_sse4.c if MATCH_SZ != 13"
+#endif
+
+/* Compute corr(im1, im2) * MATCH_SZ * stddev(im1), where the
+correlation/standard deviation are taken over MATCH_SZ by MATCH_SZ windows
+of each image, centered at (x1, y1) and (x2, y2) respectively.
+*/
+double compute_cross_correlation_avx2(unsigned char *im1, int stride1, int x1,
+ int y1, unsigned char *im2, int stride2,
+ int x2, int y2) {
+ int i, stride1_i = 0, stride2_i = 0;
+ __m256i temp1, sum_vec, sumsq2_vec, cross_vec, v, v1_1, v2_1;
+ const __m128i mask = _mm_load_si128((__m128i *)byte_mask);
+ const __m256i zero = _mm256_setzero_si256();
+ __m128i v1, v2;
+
+ sum_vec = zero;
+ sumsq2_vec = zero;
+ cross_vec = zero;
+
+ im1 += (y1 - MATCH_SZ_BY2) * stride1 + (x1 - MATCH_SZ_BY2);
+ im2 += (y2 - MATCH_SZ_BY2) * stride2 + (x2 - MATCH_SZ_BY2);
+
+ for (i = 0; i < MATCH_SZ; ++i) {
+ v1 = _mm_and_si128(_mm_loadu_si128((__m128i *)&im1[stride1_i]), mask);
+ v1_1 = _mm256_cvtepu8_epi16(v1);
+ v2 = _mm_and_si128(_mm_loadu_si128((__m128i *)&im2[stride2_i]), mask);
+ v2_1 = _mm256_cvtepu8_epi16(v2);
+
+ v = _mm256_insertf128_si256(_mm256_castsi128_si256(v1), v2, 1);
+ sumsq2_vec = _mm256_add_epi32(sumsq2_vec, _mm256_madd_epi16(v2_1, v2_1));
+
+ sum_vec = _mm256_add_epi16(sum_vec, _mm256_sad_epu8(v, zero));
+ cross_vec = _mm256_add_epi32(cross_vec, _mm256_madd_epi16(v1_1, v2_1));
+ stride1_i += stride1;
+ stride2_i += stride2;
+ }
+ __m256i sum_vec1 = _mm256_srli_si256(sum_vec, 8);
+ sum_vec = _mm256_add_epi32(sum_vec, sum_vec1);
+ int sum1_acc = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_vec));
+ int sum2_acc = _mm256_extract_epi32(sum_vec, 4);
+
+ __m256i unp_low = _mm256_unpacklo_epi64(sumsq2_vec, cross_vec);
+ __m256i unp_hig = _mm256_unpackhi_epi64(sumsq2_vec, cross_vec);
+ temp1 = _mm256_add_epi32(unp_low, unp_hig);
+
+ __m128i low_sumsq = _mm256_castsi256_si128(temp1);
+ low_sumsq = _mm_add_epi32(low_sumsq, _mm256_extractf128_si256(temp1, 1));
+ low_sumsq = _mm_add_epi32(low_sumsq, _mm_srli_epi64(low_sumsq, 32));
+ int sumsq2_acc = _mm_cvtsi128_si32(low_sumsq);
+ int cross_acc = _mm_extract_epi32(low_sumsq, 2);
+
+ int var2 = sumsq2_acc * MATCH_SZ_SQ - sum2_acc * sum2_acc;
+ int cov = cross_acc * MATCH_SZ_SQ - sum1_acc * sum2_acc;
+ return cov / sqrt((double)var2);
+}
diff --git a/test/corner_match_test.cc b/test/corner_match_test.cc
index 58e3139..af2baa7 100644
--- a/test/corner_match_test.cc
+++ b/test/corner_match_test.cc
@@ -24,9 +24,13 @@
using libaom_test::ACMRandom;
+typedef double (*ComputeCrossCorrFunc)(unsigned char *im1, int stride1, int x1,
+ int y1, unsigned char *im2, int stride2,
+ int x2, int y2);
+
using ::testing::make_tuple;
using ::testing::tuple;
-typedef tuple<int> CornerMatchParam;
+typedef tuple<int, ComputeCrossCorrFunc> CornerMatchParam;
class AV1CornerMatchTest : public ::testing::TestWithParam<CornerMatchParam> {
public:
@@ -36,19 +40,24 @@
virtual void TearDown();
protected:
- void RunCheckOutput();
+ void RunCheckOutput(int run_times);
+ ComputeCrossCorrFunc target_func;
libaom_test::ACMRandom rnd_;
};
AV1CornerMatchTest::~AV1CornerMatchTest() {}
-void AV1CornerMatchTest::SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); }
+void AV1CornerMatchTest::SetUp() {
+ rnd_.Reset(ACMRandom::DeterministicSeed());
+ target_func = GET_PARAM(1);
+}
void AV1CornerMatchTest::TearDown() { libaom_test::ClearSystemState(); }
-void AV1CornerMatchTest::RunCheckOutput() {
+void AV1CornerMatchTest::RunCheckOutput(int run_times) {
const int w = 128, h = 128;
const int num_iters = 10000;
int i, j;
+ aom_usec_timer ref_timer, test_timer;
uint8_t *input1 = new uint8_t[w * h];
uint8_t *input2 = new uint8_t[w * h];
@@ -80,21 +89,54 @@
double res_c =
compute_cross_correlation_c(input1, w, x1, y1, input2, w, x2, y2);
- double res_sse4 =
- compute_cross_correlation_sse4_1(input1, w, x1, y1, input2, w, x2, y2);
+ double res_simd = target_func(input1, w, x1, y1, input2, w, x2, y2);
- ASSERT_EQ(res_sse4, res_c);
+ if (run_times > 1) {
+ aom_usec_timer_start(&ref_timer);
+ for (j = 0; j < run_times; j++) {
+ compute_cross_correlation_c(input1, w, x1, y1, input2, w, x2, y2);
+ }
+ aom_usec_timer_mark(&ref_timer);
+ const int elapsed_time_c =
+ static_cast<int>(aom_usec_timer_elapsed(&ref_timer));
+
+ aom_usec_timer_start(&test_timer);
+ for (j = 0; j < run_times; j++) {
+ target_func(input1, w, x1, y1, input2, w, x2, y2);
+ }
+ aom_usec_timer_mark(&test_timer);
+ const int elapsed_time_simd =
+ static_cast<int>(aom_usec_timer_elapsed(&test_timer));
+
+ printf(
+ "c_time=%d \t simd_time=%d \t "
+ "gain=%d\n",
+ elapsed_time_c, elapsed_time_simd,
+ (elapsed_time_c / elapsed_time_simd));
+ } else {
+ ASSERT_EQ(res_simd, res_c);
+ }
}
-
delete[] input1;
delete[] input2;
}
-TEST_P(AV1CornerMatchTest, CheckOutput) { RunCheckOutput(); }
+TEST_P(AV1CornerMatchTest, CheckOutput) { RunCheckOutput(1); }
+TEST_P(AV1CornerMatchTest, DISABLED_Speed) { RunCheckOutput(100000); }
-INSTANTIATE_TEST_CASE_P(SSE4_1, AV1CornerMatchTest,
- ::testing::Values(make_tuple(0), make_tuple(1)));
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_CASE_P(
+ SSE4_1, AV1CornerMatchTest,
+ ::testing::Values(make_tuple(0, compute_cross_correlation_sse4_1),
+ make_tuple(1, compute_cross_correlation_sse4_1)));
+#endif
+#if HAVE_AVX2
+INSTANTIATE_TEST_CASE_P(
+ AVX2, AV1CornerMatchTest,
+ ::testing::Values(make_tuple(0, compute_cross_correlation_avx2),
+ make_tuple(1, compute_cross_correlation_avx2)));
+#endif
} // namespace AV1CornerMatch
} // namespace test_libaom
