Optimize av1_highbd_block_error module
Added AVX2 variant for av1_highbd_block_error_sse2
When tested for multiple test cases observed 0.8%
reduction in encoder time averaged across presets 1 to 4.
Module level gains improved by a factor of ~2x
on average w.r.t to SSE2 module.
Change-Id: I173503bf2d952a0cd22f20dc998e3199121ad275
diff --git a/av1/av1.cmake b/av1/av1.cmake
index e99a2ed..2c83b38 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -290,6 +290,7 @@
"${AOM_ROOT}/av1/encoder/x86/av1_highbd_quantize_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/corner_match_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c"
+ "${AOM_ROOT}/av1/encoder/x86/highbd_block_error_intrin_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm_avx2.h"
"${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm2d_avx2.c"
"${AOM_ROOT}/av1/encoder/x86/highbd_fwd_txfm_avx2.c"
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 3285fd3..e37f4d6 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -264,7 +264,7 @@
# ENCODEMB INVOKE
add_proto qw/int64_t av1_highbd_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz, int bd";
- specialize qw/av1_highbd_block_error sse2/;
+ specialize qw/av1_highbd_block_error sse2 avx2/;
# add_proto qw/void av1_highbd_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count";
diff --git a/av1/encoder/x86/highbd_block_error_intrin_avx2.c b/av1/encoder/x86/highbd_block_error_intrin_avx2.c
new file mode 100644
index 0000000..719734c
--- /dev/null
+++ b/av1/encoder/x86/highbd_block_error_intrin_avx2.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2019, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <immintrin.h>
+#include <stdio.h>
+#include "aom/aom_integer.h"
+#include "av1/common/common.h"
+
+int64_t av1_highbd_block_error_avx2(tran_low_t *coeff, tran_low_t *dqcoeff,
+ intptr_t block_size, int64_t *ssz,
+ int bps) {
+ int i;
+ int64_t temp1[8];
+ int64_t error = 0, sqcoeff = 0;
+ const int shift = 2 * (bps - 8);
+ const int rounding = shift > 0 ? 1 << (shift - 1) : 0;
+
+ for (i = 0; i < block_size; i += 16) {
+ __m256i mm256_coeff = _mm256_loadu_si256((__m256i *)(coeff + i));
+ __m256i mm256_coeff2 = _mm256_loadu_si256((__m256i *)(coeff + i + 8));
+ __m256i mm256_dqcoeff = _mm256_loadu_si256((__m256i *)(dqcoeff + i));
+ __m256i mm256_dqcoeff2 = _mm256_loadu_si256((__m256i *)(dqcoeff + i + 8));
+
+ __m256i diff1 = _mm256_sub_epi32(mm256_coeff, mm256_dqcoeff);
+ __m256i diff2 = _mm256_sub_epi32(mm256_coeff2, mm256_dqcoeff2);
+ __m256i diff1h = _mm256_srli_epi64(diff1, 32);
+ __m256i diff2h = _mm256_srli_epi64(diff2, 32);
+ __m256i res = _mm256_mul_epi32(diff1, diff1);
+ __m256i res1 = _mm256_mul_epi32(diff1h, diff1h);
+ __m256i res2 = _mm256_mul_epi32(diff2, diff2);
+ __m256i res3 = _mm256_mul_epi32(diff2h, diff2h);
+ __m256i res_diff = _mm256_add_epi64(_mm256_add_epi64(res, res1),
+ _mm256_add_epi64(res2, res3));
+ __m256i mm256_coeffh = _mm256_srli_epi64(mm256_coeff, 32);
+ __m256i mm256_coeffh2 = _mm256_srli_epi64(mm256_coeff2, 32);
+ res = _mm256_mul_epi32(mm256_coeff, mm256_coeff);
+ res1 = _mm256_mul_epi32(mm256_coeffh, mm256_coeffh);
+ res2 = _mm256_mul_epi32(mm256_coeff2, mm256_coeff2);
+ res3 = _mm256_mul_epi32(mm256_coeffh2, mm256_coeffh2);
+ __m256i res_sqcoeff = _mm256_add_epi64(_mm256_add_epi64(res, res1),
+ _mm256_add_epi64(res2, res3));
+ _mm256_storeu_si256((__m256i *)temp1, res_diff);
+ _mm256_storeu_si256((__m256i *)temp1 + 1, res_sqcoeff);
+
+ error += temp1[0] + temp1[1] + temp1[2] + temp1[3];
+ sqcoeff += temp1[4] + temp1[5] + temp1[6] + temp1[7];
+ }
+ assert(error >= 0 && sqcoeff >= 0);
+ error = (error + rounding) >> shift;
+ sqcoeff = (sqcoeff + rounding) >> shift;
+
+ *ssz = sqcoeff;
+ return error;
+}
diff --git a/test/error_block_test.cc b/test/error_block_test.cc
index 353947c..3664ccf 100644
--- a/test/error_block_test.cc
+++ b/test/error_block_test.cc
@@ -156,6 +156,70 @@
<< "First failed at test case " << first_failure;
}
+TEST_P(ErrorBlockTest, DISABLED_Speed) {
+ ACMRandom rnd(ACMRandom::DeterministicSeed());
+ DECLARE_ALIGNED(16, tran_low_t, coeff[4096]);
+ DECLARE_ALIGNED(16, tran_low_t, dqcoeff[4096]);
+ intptr_t block_size;
+ int64_t ssz;
+ int num_iters = 100000;
+ int64_t ref_ssz;
+ int k;
+ const int msb = bit_depth_ + 8 - 1;
+ for (int i = 0; i < 9; ++i) {
+ block_size = 16 << (i % 9); // All block sizes from 4x4, 8x4 ..64x64
+ for (k = 0; k < 9; k++) {
+ for (int j = 0; j < block_size; j++) {
+ if (k < 5) {
+ if (rnd(2)) {
+ // Positive number
+ coeff[j] = rnd(1 << msb);
+ dqcoeff[j] = rnd(1 << msb);
+ } else {
+ // Negative number
+ coeff[j] = -rnd(1 << msb);
+ dqcoeff[j] = -rnd(1 << msb);
+ }
+ } else {
+ if (rnd(2)) {
+ // Positive number
+ coeff[j] = rnd(1 << 14);
+ dqcoeff[j] = rnd(1 << 14);
+ } else {
+ // Negative number
+ coeff[j] = -rnd(1 << 14);
+ dqcoeff[j] = -rnd(1 << 14);
+ }
+ }
+ }
+ aom_usec_timer ref_timer, test_timer;
+
+ aom_usec_timer_start(&ref_timer);
+ for (int i = 0; i < num_iters; ++i) {
+ ref_error_block_op_(coeff, dqcoeff, block_size, &ref_ssz, bit_depth_);
+ }
+ aom_usec_timer_mark(&ref_timer);
+ const int elapsed_time_c =
+ static_cast<int>(aom_usec_timer_elapsed(&ref_timer));
+
+ aom_usec_timer_start(&test_timer);
+ for (int i = 0; i < num_iters; ++i) {
+ error_block_op_(coeff, dqcoeff, block_size, &ssz, bit_depth_);
+ }
+ aom_usec_timer_mark(&test_timer);
+
+ const int elapsed_time_simd =
+ static_cast<int>(aom_usec_timer_elapsed(&test_timer));
+
+ printf(
+ " c_time=%d \t simd_time=%d \t "
+ "gain=%d \n",
+ elapsed_time_c, elapsed_time_simd,
+ (elapsed_time_c / elapsed_time_simd));
+ }
+ }
+}
+
#if (HAVE_SSE2 || HAVE_AVX)
using ::testing::make_tuple;
@@ -168,4 +232,17 @@
make_tuple(&av1_highbd_block_error_sse2,
&av1_highbd_block_error_c, AOM_BITS_8)));
#endif // HAVE_SSE2
+
+#if (HAVE_AVX2)
+using ::testing::make_tuple;
+
+INSTANTIATE_TEST_CASE_P(
+ AVX2, ErrorBlockTest,
+ ::testing::Values(make_tuple(&av1_highbd_block_error_avx2,
+ &av1_highbd_block_error_c, AOM_BITS_10),
+ make_tuple(&av1_highbd_block_error_avx2,
+ &av1_highbd_block_error_c, AOM_BITS_12),
+ make_tuple(&av1_highbd_block_error_avx2,
+ &av1_highbd_block_error_c, AOM_BITS_8)));
+#endif // HAVE_AVX2
} // namespace
