Google Git
Sign in
aomedia/aom/d561eaddeb28c0908a5ce342fcc9e2407c349477^!/.
commit
d561eaddeb28c0908a5ce342fcc9e2407c349477
[log]
author
Yi Luo <luoyi@google.com>
Wed Jun 14 16:27:25 2017 -0700
committer
Yi Luo <luoyi@google.com>
Fri Jun 16 16:14:26 2017 +0000
tree
e2b9d76214aab330bd420a1f0c6a78a90bcaee5b
parent
1d1df182eec7b33d1041f34745cbd5a0d3263215 [diff]
Add avx2 fast path quantizer 32x32

- User level encoder time reduction ~2.3% on i7-6700.

Change-Id: I04a5cf44c398b295977ff8c8958471a896bfb20d

diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index ac857cb..ef6aeee 100755
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl

@@ -323,6 +323,7 @@
     specialize qw/av1_quantize_fp sse2 avx2/;
 
     add_proto qw/void av1_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
+    specialize qw/av1_quantize_fp_32x32 avx2/;
 
     if (aom_config("CONFIG_TX64X64") eq "yes") {
       add_proto qw/void av1_quantize_fp_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
@@ -340,7 +341,7 @@
     specialize qw/av1_quantize_fp neon sse2 avx2/, "$ssse3_x86_64";
 
     add_proto qw/void av1_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
-    specialize qw/av1_quantize_fp_32x32/, "$ssse3_x86_64";
+    specialize qw/av1_quantize_fp_32x32 avx2/, "$ssse3_x86_64";
 
     if (aom_config("CONFIG_TX64X64") eq "yes") {
       add_proto qw/void av1_quantize_fp_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";

diff --git a/av1/encoder/x86/av1_quantize_avx2.c b/av1/encoder/x86/av1_quantize_avx2.c
index d9302d7..1c0a120 100644
--- a/av1/encoder/x86/av1_quantize_avx2.c
+++ b/av1/encoder/x86/av1_quantize_avx2.c

@@ -44,12 +44,23 @@
 static INLINE void init_qp(const int16_t *round_ptr, const int16_t *quant_ptr,
                            const int16_t *dequant_ptr, int log_scale,
                            __m256i *thr, __m256i *qp) {
-  const __m128i round = _mm_loadu_si128((const __m128i *)round_ptr);
+  __m128i round = _mm_loadu_si128((const __m128i *)round_ptr);
   const __m128i quant = _mm_loadu_si128((const __m128i *)quant_ptr);
   const __m128i dequant = _mm_loadu_si128((const __m128i *)dequant_ptr);
 
+  if (log_scale > 0) {
+    const __m128i rnd = _mm_set1_epi16((int16_t)1 << (log_scale - 1));
+    round = _mm_add_epi16(round, rnd);
+    round = _mm_srai_epi16(round, log_scale);
+  }
+
   init_one_qp(&round, &qp[0]);
   init_one_qp(&quant, &qp[1]);
+
+  if (log_scale > 0) {
+    qp[1] = _mm256_slli_epi16(qp[1], log_scale);
+  }
+
   init_one_qp(&dequant, &qp[2]);
   *thr = _mm256_srai_epi16(qp[2], 1 + log_scale);
 }
@@ -61,11 +72,37 @@
   *thr = _mm256_srai_epi16(qp[2], 1 + log_scale);
 }
 
+#define store_quan(q, addr)                               \
+  do {                                                    \
+    __m256i sign_bits = _mm256_srai_epi16(q, 15);         \
+    __m256i y0 = _mm256_unpacklo_epi16(q, sign_bits);     \
+    __m256i y1 = _mm256_unpackhi_epi16(q, sign_bits);     \
+    __m256i x0 = _mm256_permute2x128_si256(y0, y1, 0x20); \
+    __m256i x1 = _mm256_permute2x128_si256(y0, y1, 0x31); \
+    _mm256_storeu_si256((__m256i *)addr, x0);             \
+    _mm256_storeu_si256((__m256i *)addr + 1, x1);         \
+  } while (0)
+
+#if CONFIG_HIGHBITDEPTH
+#define store_two_quan(q, addr1, dq, addr2) \
+  do {                                      \
+    store_quan(q, addr1);                   \
+    store_quan(dq, addr2);                  \
+  } while (0)
+#else
+#define store_two_quan(q, addr1, dq, addr2)    \
+  do {                                         \
+    _mm256_storeu_si256((__m256i *)addr1, q);  \
+    _mm256_storeu_si256((__m256i *)addr2, dq); \
+  } while (0)
+#endif
+
 static INLINE void quantize(const __m256i *thr, const __m256i *qp, __m256i *c,
                             const int16_t *iscan_ptr, tran_low_t *qcoeff,
                             tran_low_t *dqcoeff, __m256i *eob) {
   const __m256i abs = _mm256_abs_epi16(*c);
-  const __m256i mask = _mm256_cmpgt_epi16(abs, *thr);
+  __m256i mask = _mm256_cmpgt_epi16(abs, *thr);
+  mask = _mm256_or_si256(mask, _mm256_cmpeq_epi16(abs, *thr));
   const int nzflag = _mm256_movemask_epi8(mask);
 
   if (nzflag) {
@@ -74,27 +111,7 @@
     q = _mm256_sign_epi16(q, *c);
     const __m256i dq = _mm256_mullo_epi16(q, qp[2]);
 
-#if CONFIG_HIGHBITDEPTH
-    __m256i sign_bits = _mm256_srai_epi16(q, 15);
-    __m256i y0 = _mm256_unpacklo_epi16(q, sign_bits);
-    __m256i y1 = _mm256_unpackhi_epi16(q, sign_bits);
-    __m256i x0 = _mm256_permute2x128_si256(y0, y1, 0x20);
-    __m256i x1 = _mm256_permute2x128_si256(y0, y1, 0x31);
-    _mm256_storeu_si256((__m256i *)qcoeff, x0);
-    _mm256_storeu_si256((__m256i *)qcoeff + 1, x1);
-
-    sign_bits = _mm256_srai_epi16(dq, 15);
-    y0 = _mm256_unpacklo_epi16(dq, sign_bits);
-    y1 = _mm256_unpackhi_epi16(dq, sign_bits);
-    x0 = _mm256_permute2x128_si256(y0, y1, 0x20);
-    x1 = _mm256_permute2x128_si256(y0, y1, 0x31);
-    _mm256_storeu_si256((__m256i *)dqcoeff, x0);
-    _mm256_storeu_si256((__m256i *)dqcoeff + 1, x1);
-#else
-    _mm256_storeu_si256((__m256i *)qcoeff, q);
-    _mm256_storeu_si256((__m256i *)dqcoeff, dq);
-#endif
-
+    store_two_quan(q, qcoeff, dq, dqcoeff);
     const __m256i zero = _mm256_setzero_si256();
     const __m256i iscan = _mm256_loadu_si256((const __m256i *)iscan_ptr);
     const __m256i zero_coeff = _mm256_cmpeq_epi16(dq, zero);
@@ -172,3 +189,101 @@
     *eob_ptr = 0;
   }
 }
+
+static INLINE void quantize_32x32(const __m256i *thr, const __m256i *qp,
+                                  __m256i *c, const int16_t *iscan_ptr,
+                                  tran_low_t *qcoeff, tran_low_t *dqcoeff,
+                                  __m256i *eob) {
+  const __m256i abs = _mm256_abs_epi16(*c);
+  __m256i mask = _mm256_cmpgt_epi16(abs, *thr);
+  mask = _mm256_or_si256(mask, _mm256_cmpeq_epi16(abs, *thr));
+  const int nzflag = _mm256_movemask_epi8(mask);
+
+  if (nzflag) {
+    __m256i q = _mm256_adds_epi16(abs, qp[0]);
+    q = _mm256_mulhi_epu16(q, qp[1]);
+
+    __m256i dq = _mm256_mullo_epi16(q, qp[2]);
+    dq = _mm256_srli_epi16(dq, 1);
+
+    q = _mm256_sign_epi16(q, *c);
+    dq = _mm256_sign_epi16(dq, *c);
+
+    store_two_quan(q, qcoeff, dq, dqcoeff);
+    const __m256i zero = _mm256_setzero_si256();
+    const __m256i iscan = _mm256_loadu_si256((const __m256i *)iscan_ptr);
+    const __m256i zero_coeff = _mm256_cmpeq_epi16(dq, zero);
+    const __m256i nzero_coeff = _mm256_cmpeq_epi16(zero_coeff, zero);
+    __m256i cur_eob = _mm256_sub_epi16(iscan, nzero_coeff);
+    cur_eob = _mm256_and_si256(cur_eob, nzero_coeff);
+    *eob = _mm256_max_epi16(*eob, cur_eob);
+  } else {
+    write_zero(qcoeff);
+    write_zero(dqcoeff);
+  }
+}
+
+void av1_quantize_fp_32x32_avx2(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block,
+    const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr,
+    const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+    const int16_t *scan_ptr, const int16_t *iscan_ptr) {
+  (void)scan_ptr;
+  (void)zbin_ptr;
+  (void)quant_shift_ptr;
+  const unsigned int step = 16;
+
+  if (LIKELY(!skip_block)) {
+    __m256i qp[3];
+    __m256i coeff, thr;
+    const int log_scale = 1;
+
+    init_qp(round_ptr, quant_ptr, dequant_ptr, log_scale, &thr, qp);
+    read_coeff(coeff_ptr, &coeff);
+
+    __m256i eob = _mm256_setzero_si256();
+    quantize_32x32(&thr, qp, &coeff, iscan_ptr, qcoeff_ptr, dqcoeff_ptr, &eob);
+
+    coeff_ptr += step;
+    qcoeff_ptr += step;
+    dqcoeff_ptr += step;
+    iscan_ptr += step;
+    n_coeffs -= step;
+
+    update_qp(log_scale, &thr, qp);
+
+    while (n_coeffs > 0) {
+      read_coeff(coeff_ptr, &coeff);
+      quantize_32x32(&thr, qp, &coeff, iscan_ptr, qcoeff_ptr, dqcoeff_ptr,
+                     &eob);
+
+      coeff_ptr += step;
+      qcoeff_ptr += step;
+      dqcoeff_ptr += step;
+      iscan_ptr += step;
+      n_coeffs -= step;
+    }
+    {
+      __m256i eob_s;
+      eob_s = _mm256_shuffle_epi32(eob, 0xe);
+      eob = _mm256_max_epi16(eob, eob_s);
+      eob_s = _mm256_shufflelo_epi16(eob, 0xe);
+      eob = _mm256_max_epi16(eob, eob_s);
+      eob_s = _mm256_shufflelo_epi16(eob, 1);
+      eob = _mm256_max_epi16(eob, eob_s);
+      const __m128i final_eob = _mm_max_epi16(_mm256_castsi256_si128(eob),
+                                              _mm256_extractf128_si256(eob, 1));
+      *eob_ptr = _mm_extract_epi16(final_eob, 0);
+    }
+  } else {
+    do {
+      write_zero(qcoeff_ptr);
+      write_zero(dqcoeff_ptr);
+      qcoeff_ptr += step;
+      dqcoeff_ptr += step;
+      n_coeffs -= step;
+    } while (n_coeffs > 0);
+    *eob_ptr = 0;
+  }
+}

diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index c600344..3275e32 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc

@@ -249,8 +249,8 @@
 #if HAVE_AVX2
 const QuantizeParam kQParamArrayAvx2[] = {
   make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X16, AOM_BITS_8),
-  // make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32,
-  //            AOM_BITS_8)
+  make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32,
+             AOM_BITS_8),
 };
 
 INSTANTIATE_TEST_CASE_P(AVX2, QuantizeTest,