Improve AVX2 intrinsic of av1_block_error_lp()
The CL optimizes av1_block_error_lp_avx2() by reducing
the occurences of expanding to higher precision while
accumulation.
Encoder speed-up for RT preset,
Instruction Count
cpu Testset Reduction(%)
7 rtc 0.712
7 rtc_derf 0.557
8 rtc 0.560
8 rtc_derf 0.426
9 rtc 0.580
9 rtc_derf 0.414
9 rtc_screen 0.529
10 rtc 0.527
10 rtc_derf 0.395
10 rtc_screen 0.529
Change-Id: I82b706897dc1c2d611d01714443fd1c2706db09b
diff --git a/av1/encoder/x86/error_intrin_avx2.c b/av1/encoder/x86/error_intrin_avx2.c
index 12dda3a..57725d1 100644
--- a/av1/encoder/x86/error_intrin_avx2.c
+++ b/av1/encoder/x86/error_intrin_avx2.c
@@ -29,53 +29,122 @@
}
}
-int64_t av1_block_error_lp_avx2(const int16_t *coeff, const int16_t *dqcoeff,
- intptr_t block_size) {
+static INLINE void av1_block_error_num_coeff16_avx2(const int16_t *coeff,
+ const int16_t *dqcoeff,
+ __m256i *sse_256) {
+ const __m256i _coeff = _mm256_loadu_si256((const __m256i *)coeff);
+ const __m256i _dqcoeff = _mm256_loadu_si256((const __m256i *)dqcoeff);
+ // d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15
+ const __m256i diff = _mm256_sub_epi16(_dqcoeff, _coeff);
+ // r0 r1 r2 r3 r4 r5 r6 r7
+ const __m256i error = _mm256_madd_epi16(diff, diff);
+ // r0+r1 r2+r3 | r0+r1 r2+r3 | r4+r5 r6+r7 | r4+r5 r6+r7
+ const __m256i error_hi = _mm256_hadd_epi32(error, error);
+ // r0+r1 | r2+r3 | r4+r5 | r6+r7
+ *sse_256 = _mm256_unpacklo_epi32(error_hi, _mm256_setzero_si256());
+}
+
+static INLINE void av1_block_error_num_coeff32_avx2(const int16_t *coeff,
+ const int16_t *dqcoeff,
+ __m256i *sse_256) {
const __m256i zero = _mm256_setzero_si256();
- __m256i sse_256 = zero;
- __m256i sse_hi;
- __m128i sse_128;
+ const __m256i _coeff_0 = _mm256_loadu_si256((const __m256i *)coeff);
+ const __m256i _dqcoeff_0 = _mm256_loadu_si256((const __m256i *)dqcoeff);
+ const __m256i _coeff_1 = _mm256_loadu_si256((const __m256i *)(coeff + 16));
+ const __m256i _dqcoeff_1 =
+ _mm256_loadu_si256((const __m256i *)(dqcoeff + 16));
+
+ // d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15
+ const __m256i diff_0 = _mm256_sub_epi16(_dqcoeff_0, _coeff_0);
+ const __m256i diff_1 = _mm256_sub_epi16(_dqcoeff_1, _coeff_1);
+
+ // r0 r1 r2 r3 r4 r5 r6 r7
+ const __m256i error_0 = _mm256_madd_epi16(diff_0, diff_0);
+ const __m256i error_1 = _mm256_madd_epi16(diff_1, diff_1);
+ const __m256i err_final_0 = _mm256_add_epi32(error_0, error_1);
+
+ // For extreme input values, the accumulation needs to happen in 64 bit
+ // precision to avoid any overflow.
+ const __m256i exp0_error_lo = _mm256_unpacklo_epi32(err_final_0, zero);
+ const __m256i exp0_error_hi = _mm256_unpackhi_epi32(err_final_0, zero);
+ const __m256i sum_temp_0 = _mm256_add_epi64(exp0_error_hi, exp0_error_lo);
+ *sse_256 = _mm256_add_epi64(*sse_256, sum_temp_0);
+}
+
+static INLINE void av1_block_error_num_coeff64_avx2(const int16_t *coeff,
+ const int16_t *dqcoeff,
+ __m256i *sse_256,
+ intptr_t num_coeff) {
+ const __m256i zero = _mm256_setzero_si256();
+ for (int i = 0; i < num_coeff; i += 64) {
+ // Load 64 elements for coeff and dqcoeff.
+ const __m256i _coeff_0 = _mm256_loadu_si256((const __m256i *)coeff);
+ const __m256i _dqcoeff_0 = _mm256_loadu_si256((const __m256i *)dqcoeff);
+ const __m256i _coeff_1 = _mm256_loadu_si256((const __m256i *)(coeff + 16));
+ const __m256i _dqcoeff_1 =
+ _mm256_loadu_si256((const __m256i *)(dqcoeff + 16));
+ const __m256i _coeff_2 = _mm256_loadu_si256((const __m256i *)(coeff + 32));
+ const __m256i _dqcoeff_2 =
+ _mm256_loadu_si256((const __m256i *)(dqcoeff + 32));
+ const __m256i _coeff_3 = _mm256_loadu_si256((const __m256i *)(coeff + 48));
+ const __m256i _dqcoeff_3 =
+ _mm256_loadu_si256((const __m256i *)(dqcoeff + 48));
+
+ // d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15
+ const __m256i diff_0 = _mm256_sub_epi16(_dqcoeff_0, _coeff_0);
+ const __m256i diff_1 = _mm256_sub_epi16(_dqcoeff_1, _coeff_1);
+ const __m256i diff_2 = _mm256_sub_epi16(_dqcoeff_2, _coeff_2);
+ const __m256i diff_3 = _mm256_sub_epi16(_dqcoeff_3, _coeff_3);
+
+ // r0 r1 r2 r3 r4 r5 r6 r7
+ const __m256i error_0 = _mm256_madd_epi16(diff_0, diff_0);
+ const __m256i error_1 = _mm256_madd_epi16(diff_1, diff_1);
+ const __m256i error_2 = _mm256_madd_epi16(diff_2, diff_2);
+ const __m256i error_3 = _mm256_madd_epi16(diff_3, diff_3);
+ // r00 r01 r02 r03 r04 r05 r06 r07
+ const __m256i err_final_0 = _mm256_add_epi32(error_0, error_1);
+ // r10 r11 r12 r13 r14 r15 r16 r17
+ const __m256i err_final_1 = _mm256_add_epi32(error_2, error_3);
+
+ // For extreme input values, the accumulation needs to happen in 64 bit
+ // precision to avoid any overflow. r00 r01 r04 r05
+ const __m256i exp0_error_lo = _mm256_unpacklo_epi32(err_final_0, zero);
+ // r02 r03 r06 r07
+ const __m256i exp0_error_hi = _mm256_unpackhi_epi32(err_final_0, zero);
+ // r10 r11 r14 r15
+ const __m256i exp1_error_lo = _mm256_unpacklo_epi32(err_final_1, zero);
+ // r12 r13 r16 r17
+ const __m256i exp1_error_hi = _mm256_unpackhi_epi32(err_final_1, zero);
+
+ const __m256i sum_temp_0 = _mm256_add_epi64(exp0_error_hi, exp0_error_lo);
+ const __m256i sum_temp_1 = _mm256_add_epi64(exp1_error_hi, exp1_error_lo);
+ const __m256i sse_256_temp = _mm256_add_epi64(sum_temp_1, sum_temp_0);
+ *sse_256 = _mm256_add_epi64(*sse_256, sse_256_temp);
+ coeff += 64;
+ dqcoeff += 64;
+ }
+}
+
+int64_t av1_block_error_lp_avx2(const int16_t *coeff, const int16_t *dqcoeff,
+ intptr_t num_coeff) {
+ assert(num_coeff % 16 == 0);
+ __m256i sse_256 = _mm256_setzero_si256();
int64_t sse;
- if (block_size == 16) {
- // Load 16 elements for coeff and dqcoeff.
- const __m256i _coeff = _mm256_loadu_si256((const __m256i *)coeff);
- const __m256i _dqcoeff = _mm256_loadu_si256((const __m256i *)dqcoeff);
- // dqcoeff - coeff
- const __m256i diff = _mm256_sub_epi16(_dqcoeff, _coeff);
- // madd (dqcoeff - coeff)
- const __m256i error_lo = _mm256_madd_epi16(diff, diff);
- // Save the higher 64 bit of each 128 bit lane.
- const __m256i error_hi = _mm256_srli_si256(error_lo, 8);
- // Add the higher 64 bit to the low 64 bit.
- const __m256i error = _mm256_add_epi32(error_lo, error_hi);
- // Expand each double word in the lower 64 bits to quad word.
- sse_256 = _mm256_unpacklo_epi32(error, zero);
- } else {
- for (int i = 0; i < block_size; i += 16) {
- // Load 16 elements for coeff and dqcoeff.
- const __m256i _coeff = _mm256_loadu_si256((const __m256i *)coeff);
- const __m256i _dqcoeff = _mm256_loadu_si256((const __m256i *)dqcoeff);
- const __m256i diff = _mm256_sub_epi16(_dqcoeff, _coeff);
- const __m256i error = _mm256_madd_epi16(diff, diff);
- // Expand each double word of madd (dqcoeff - coeff) to quad word.
- const __m256i exp_error_lo = _mm256_unpacklo_epi32(error, zero);
- const __m256i exp_error_hi = _mm256_unpackhi_epi32(error, zero);
- // Add each quad word of madd (dqcoeff - coeff).
- sse_256 = _mm256_add_epi64(sse_256, exp_error_lo);
- sse_256 = _mm256_add_epi64(sse_256, exp_error_hi);
- coeff += 16;
- dqcoeff += 16;
- }
- }
+ if (num_coeff == 16)
+ av1_block_error_num_coeff16_avx2(coeff, dqcoeff, &sse_256);
+ else if (num_coeff == 32)
+ av1_block_error_num_coeff32_avx2(coeff, dqcoeff, &sse_256);
+ else
+ av1_block_error_num_coeff64_avx2(coeff, dqcoeff, &sse_256, num_coeff);
+
// Save the higher 64 bit of each 128 bit lane.
- sse_hi = _mm256_srli_si256(sse_256, 8);
+ const __m256i sse_hi = _mm256_srli_si256(sse_256, 8);
// Add the higher 64 bit to the low 64 bit.
sse_256 = _mm256_add_epi64(sse_256, sse_hi);
-
- // Add each 64 bit from each of the 128 bit lane of the 256 bit.
- sse_128 = _mm_add_epi64(_mm256_castsi256_si128(sse_256),
- _mm256_extractf128_si256(sse_256, 1));
+ // Accumulate the sse_256 register to get final sse
+ const __m128i sse_128 = _mm_add_epi64(_mm256_castsi256_si128(sse_256),
+ _mm256_extractf128_si256(sse_256, 1));
// Store the results.
_mm_storel_epi64((__m128i *)&sse, sse_128);
