Add AVX2 optimization for cross_correlation module Added compute_cross_correlation avx2 function and changed test_bench to support AVX2 version. Module level gains improved by a factor of ~1.45 on average w.r.t to SSE4_1 modules. When tested for multiple test cases observed 0.4% average reduction in encoder time for speed = 1 preset. Change-Id: I7ee5d26d15b3d0bd36f5def64d0af58079ec604a
diff --git a/av1/av1.cmake b/av1/av1.cmake index 8c92615..19af0ff 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -282,6 +282,7 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_AVX2 "${AOM_ROOT}/av1/encoder/x86/av1_quantize_avx2.c" "${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_avx2.c" + "${AOM_ROOT}/av1/encoder/x86/corner_match_avx2.c" "${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c" "${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm_avx2.h" "${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm2d_avx2.c"
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index aa7fddf..e0ca13f 100755 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -336,7 +336,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") { add_proto qw/double compute_cross_correlation/, "unsigned char *im1, int stride1, int x1, int y1, unsigned char *im2, int stride2, int x2, int y2"; - specialize qw/compute_cross_correlation sse4_1/; + specialize qw/compute_cross_correlation sse4_1 avx2/; } # LOOP_RESTORATION functions
diff --git a/av1/encoder/x86/corner_match_avx2.c b/av1/encoder/x86/corner_match_avx2.c new file mode 100644 index 0000000..3351d0a --- /dev/null +++ b/av1/encoder/x86/corner_match_avx2.c
@@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <stdlib.h> +#include <memory.h> +#include <math.h> +#include <assert.h> + +#include <immintrin.h> +#include "config/av1_rtcd.h" + +#include "aom_ports/mem.h" +#include "av1/encoder/corner_match.h" + +DECLARE_ALIGNED(16, static const uint8_t, byte_mask[16]) = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0 +}; +#if MATCH_SZ != 13 +#error "Need to change byte_mask in corner_match_sse4.c if MATCH_SZ != 13" +#endif + +/* Compute corr(im1, im2) * MATCH_SZ * stddev(im1), where the +correlation/standard deviation are taken over MATCH_SZ by MATCH_SZ windows +of each image, centered at (x1, y1) and (x2, y2) respectively. +*/ +double compute_cross_correlation_avx2(unsigned char *im1, int stride1, int x1, + int y1, unsigned char *im2, int stride2, + int x2, int y2) { + int i, stride1_i = 0, stride2_i = 0; + __m256i temp1, sum_vec, sumsq2_vec, cross_vec, v, v1_1, v2_1; + const __m128i mask = _mm_load_si128((__m128i *)byte_mask); + const __m256i zero = _mm256_setzero_si256(); + __m128i v1, v2; + + sum_vec = zero; + sumsq2_vec = zero; + cross_vec = zero; + + im1 += (y1 - MATCH_SZ_BY2) * stride1 + (x1 - MATCH_SZ_BY2); + im2 += (y2 - MATCH_SZ_BY2) * stride2 + (x2 - MATCH_SZ_BY2); + + for (i = 0; i < MATCH_SZ; ++i) { + v1 = _mm_and_si128(_mm_loadu_si128((__m128i *)&im1[stride1_i]), mask); + v1_1 = _mm256_cvtepu8_epi16(v1); + v2 = _mm_and_si128(_mm_loadu_si128((__m128i *)&im2[stride2_i]), mask); + v2_1 = _mm256_cvtepu8_epi16(v2); + + v = _mm256_insertf128_si256(_mm256_castsi128_si256(v1), v2, 1); + sumsq2_vec = _mm256_add_epi32(sumsq2_vec, _mm256_madd_epi16(v2_1, v2_1)); + + sum_vec = _mm256_add_epi16(sum_vec, _mm256_sad_epu8(v, zero)); + cross_vec = _mm256_add_epi32(cross_vec, _mm256_madd_epi16(v1_1, v2_1)); + stride1_i += stride1; + stride2_i += stride2; + } + __m256i sum_vec1 = _mm256_srli_si256(sum_vec, 8); + sum_vec = _mm256_add_epi32(sum_vec, sum_vec1); + int sum1_acc = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_vec)); + int sum2_acc = _mm256_extract_epi32(sum_vec, 4); + + __m256i unp_low = _mm256_unpacklo_epi64(sumsq2_vec, cross_vec); + __m256i unp_hig = _mm256_unpackhi_epi64(sumsq2_vec, cross_vec); + temp1 = _mm256_add_epi32(unp_low, unp_hig); + + __m128i low_sumsq = _mm256_castsi256_si128(temp1); + low_sumsq = _mm_add_epi32(low_sumsq, _mm256_extractf128_si256(temp1, 1)); + low_sumsq = _mm_add_epi32(low_sumsq, _mm_srli_epi64(low_sumsq, 32)); + int sumsq2_acc = _mm_cvtsi128_si32(low_sumsq); + int cross_acc = _mm_extract_epi32(low_sumsq, 2); + + int var2 = sumsq2_acc * MATCH_SZ_SQ - sum2_acc * sum2_acc; + int cov = cross_acc * MATCH_SZ_SQ - sum1_acc * sum2_acc; + return cov / sqrt((double)var2); +}
diff --git a/test/corner_match_test.cc b/test/corner_match_test.cc index 58e3139..af2baa7 100644 --- a/test/corner_match_test.cc +++ b/test/corner_match_test.cc
@@ -24,9 +24,13 @@ using libaom_test::ACMRandom; +typedef double (*ComputeCrossCorrFunc)(unsigned char *im1, int stride1, int x1, + int y1, unsigned char *im2, int stride2, + int x2, int y2); + using ::testing::make_tuple; using ::testing::tuple; -typedef tuple<int> CornerMatchParam; +typedef tuple<int, ComputeCrossCorrFunc> CornerMatchParam; class AV1CornerMatchTest : public ::testing::TestWithParam<CornerMatchParam> { public: @@ -36,19 +40,24 @@ virtual void TearDown(); protected: - void RunCheckOutput(); + void RunCheckOutput(int run_times); + ComputeCrossCorrFunc target_func; libaom_test::ACMRandom rnd_; }; AV1CornerMatchTest::~AV1CornerMatchTest() {} -void AV1CornerMatchTest::SetUp() { rnd_.Reset(ACMRandom::DeterministicSeed()); } +void AV1CornerMatchTest::SetUp() { + rnd_.Reset(ACMRandom::DeterministicSeed()); + target_func = GET_PARAM(1); +} void AV1CornerMatchTest::TearDown() { libaom_test::ClearSystemState(); } -void AV1CornerMatchTest::RunCheckOutput() { +void AV1CornerMatchTest::RunCheckOutput(int run_times) { const int w = 128, h = 128; const int num_iters = 10000; int i, j; + aom_usec_timer ref_timer, test_timer; uint8_t *input1 = new uint8_t[w * h]; uint8_t *input2 = new uint8_t[w * h]; @@ -80,21 +89,54 @@ double res_c = compute_cross_correlation_c(input1, w, x1, y1, input2, w, x2, y2); - double res_sse4 = - compute_cross_correlation_sse4_1(input1, w, x1, y1, input2, w, x2, y2); + double res_simd = target_func(input1, w, x1, y1, input2, w, x2, y2); - ASSERT_EQ(res_sse4, res_c); + if (run_times > 1) { + aom_usec_timer_start(&ref_timer); + for (j = 0; j < run_times; j++) { + compute_cross_correlation_c(input1, w, x1, y1, input2, w, x2, y2); + } + aom_usec_timer_mark(&ref_timer); + const int elapsed_time_c = + static_cast<int>(aom_usec_timer_elapsed(&ref_timer)); + + aom_usec_timer_start(&test_timer); + for (j = 0; j < run_times; j++) { + target_func(input1, w, x1, y1, input2, w, x2, y2); + } + aom_usec_timer_mark(&test_timer); + const int elapsed_time_simd = + static_cast<int>(aom_usec_timer_elapsed(&test_timer)); + + printf( + "c_time=%d \t simd_time=%d \t " + "gain=%d\n", + elapsed_time_c, elapsed_time_simd, + (elapsed_time_c / elapsed_time_simd)); + } else { + ASSERT_EQ(res_simd, res_c); + } } - delete[] input1; delete[] input2; } -TEST_P(AV1CornerMatchTest, CheckOutput) { RunCheckOutput(); } +TEST_P(AV1CornerMatchTest, CheckOutput) { RunCheckOutput(1); } +TEST_P(AV1CornerMatchTest, DISABLED_Speed) { RunCheckOutput(100000); } -INSTANTIATE_TEST_CASE_P(SSE4_1, AV1CornerMatchTest, - ::testing::Values(make_tuple(0), make_tuple(1))); +#if HAVE_SSE4_1 +INSTANTIATE_TEST_CASE_P( + SSE4_1, AV1CornerMatchTest, + ::testing::Values(make_tuple(0, compute_cross_correlation_sse4_1), + make_tuple(1, compute_cross_correlation_sse4_1))); +#endif +#if HAVE_AVX2 +INSTANTIATE_TEST_CASE_P( + AVX2, AV1CornerMatchTest, + ::testing::Values(make_tuple(0, compute_cross_correlation_avx2), + make_tuple(1, compute_cross_correlation_avx2))); +#endif } // namespace AV1CornerMatch } // namespace test_libaom