Add avx2 highbd_quantize_b - First pass encoding time reduces ~10.9% on i7-6700 at 100 frames, 1080p. - avx2 works for coeff number >= 8 cases; coeff number < 8 case will be implemented by sse2. - Unit test is added type B/FP/DC. Change-Id: Ibe5b7807c64e6dfc2d59c470ed50a6e8ca94ef7c
diff --git a/aom_dsp/aom_dsp.mk b/aom_dsp/aom_dsp.mk index 6e2d563..2274369 100644 --- a/aom_dsp/aom_dsp.mk +++ b/aom_dsp/aom_dsp.mk
@@ -292,6 +292,7 @@ DSP_SRCS-$(HAVE_SSE2) += x86/quantize_sse2.c ifeq ($(CONFIG_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_SSE2) += x86/highbd_quantize_intrin_sse2.c +DSP_SRCS-$(HAVE_AVX2) += x86/highbd_quantize_intrin_avx2.c endif ifeq ($(ARCH_X86_64),yes) DSP_SRCS-$(HAVE_SSSE3) += x86/quantize_ssse3_x86_64.asm
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 36c960d..6b0a46e 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -545,7 +545,7 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") { add_proto qw/void aom_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - specialize qw/aom_highbd_quantize_b sse2/; + specialize qw/aom_highbd_quantize_b sse2 avx2/; add_proto qw/void aom_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/aom_highbd_quantize_b_32x32 sse2/;
diff --git a/aom_dsp/x86/highbd_quantize_intrin_avx2.c b/aom_dsp/x86/highbd_quantize_intrin_avx2.c new file mode 100644 index 0000000..2bbf15e --- /dev/null +++ b/aom_dsp/x86/highbd_quantize_intrin_avx2.c
@@ -0,0 +1,171 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <immintrin.h> + +#include "./aom_dsp_rtcd.h" +#include "aom/aom_integer.h" + +static INLINE void init_one_qp(const __m128i *p, __m256i *qp) { + const __m128i sign = _mm_srai_epi16(*p, 15); + const __m128i dc = _mm_unpacklo_epi16(*p, sign); + const __m128i ac = _mm_unpackhi_epi16(*p, sign); + *qp = _mm256_insertf128_si256(_mm256_castsi128_si256(dc), ac, 1); +} + +static INLINE void update_qp(__m256i *qp) { + int i; + for (i = 0; i < 5; ++i) { + qp[i] = _mm256_permute2x128_si256(qp[i], qp[i], 0x11); + } +} + +static INLINE void init_qp(const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *dequant_ptr, + const int16_t *quant_shift_ptr, __m256i *qp) { + const __m128i zbin = _mm_loadu_si128((const __m128i *)zbin_ptr); + const __m128i round = _mm_loadu_si128((const __m128i *)round_ptr); + const __m128i quant = _mm_loadu_si128((const __m128i *)quant_ptr); + const __m128i dequant = _mm_loadu_si128((const __m128i *)dequant_ptr); + const __m128i quant_shift = _mm_loadu_si128((const __m128i *)quant_shift_ptr); + init_one_qp(&zbin, &qp[0]); + init_one_qp(&round, &qp[1]); + init_one_qp(&quant, &qp[2]); + init_one_qp(&dequant, &qp[3]); + init_one_qp(&quant_shift, &qp[4]); +} + +// Note: +// *x is vector multiplied by *y which is 16 int32_t parallel multiplication +// and right shift 16. The output, 16 int32_t is save in *p. +static INLINE void mm256_mul_shift_epi32(const __m256i *x, const __m256i *y, + __m256i *p) { + __m256i prod_lo = _mm256_mul_epi32(*x, *y); + __m256i prod_hi = _mm256_srli_epi64(*x, 32); + const __m256i mult_hi = _mm256_srli_epi64(*y, 32); + prod_hi = _mm256_mul_epi32(prod_hi, mult_hi); + + prod_lo = _mm256_srli_epi64(prod_lo, 16); + const __m256i mask = _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1); + prod_lo = _mm256_and_si256(prod_lo, mask); + prod_hi = _mm256_srli_epi64(prod_hi, 16); + + prod_hi = _mm256_slli_epi64(prod_hi, 32); + *p = _mm256_or_si256(prod_lo, prod_hi); +} + +static INLINE void quantize(const __m256i *qp, __m256i *c, + const int16_t *iscan_ptr, tran_low_t *qcoeff, + tran_low_t *dqcoeff, __m256i *eob) { + const __m256i abs = _mm256_abs_epi32(*c); + const __m256i flag1 = _mm256_cmpgt_epi32(abs, qp[0]); + __m256i flag2 = _mm256_cmpeq_epi32(abs, qp[0]); + flag2 = _mm256_or_si256(flag1, flag2); + const int32_t nzflag = _mm256_movemask_epi8(flag2); + + if (LIKELY(nzflag)) { + __m256i q = _mm256_add_epi32(abs, qp[1]); + __m256i tmp; + mm256_mul_shift_epi32(&q, &qp[2], &tmp); + q = _mm256_add_epi32(tmp, q); + + mm256_mul_shift_epi32(&q, &qp[4], &q); + __m256i dq = _mm256_mullo_epi32(q, qp[3]); + + q = _mm256_sign_epi32(q, *c); + dq = _mm256_sign_epi32(dq, *c); + q = _mm256_and_si256(q, flag2); + dq = _mm256_and_si256(dq, flag2); + + _mm256_storeu_si256((__m256i *)qcoeff, q); + _mm256_storeu_si256((__m256i *)dqcoeff, dq); + + const __m128i isc = _mm_loadu_si128((const __m128i *)iscan_ptr); + const __m128i zr = _mm_setzero_si128(); + const __m128i lo = _mm_unpacklo_epi16(isc, zr); + const __m128i hi = _mm_unpackhi_epi16(isc, zr); + const __m256i iscan = + _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1); + + const __m256i zero = _mm256_setzero_si256(); + const __m256i zc = _mm256_cmpeq_epi32(dq, zero); + const __m256i nz = _mm256_cmpeq_epi32(zc, zero); + __m256i cur_eob = _mm256_sub_epi32(iscan, nz); + cur_eob = _mm256_and_si256(cur_eob, nz); + *eob = _mm256_max_epi32(cur_eob, *eob); + } else { + const __m256i zero = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i *)qcoeff, zero); + _mm256_storeu_si256((__m256i *)dqcoeff, zero); + } +} + +void aom_highbd_quantize_b_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + (void)scan; + const unsigned int step = 8; + + if (LIKELY(!skip_block)) { + __m256i qp[5], coeff; + init_qp(zbin_ptr, round_ptr, quant_ptr, dequant_ptr, quant_shift_ptr, qp); + coeff = _mm256_loadu_si256((const __m256i *)coeff_ptr); + + __m256i eob = _mm256_setzero_si256(); + quantize(qp, &coeff, iscan, qcoeff_ptr, dqcoeff_ptr, &eob); + + coeff_ptr += step; + qcoeff_ptr += step; + dqcoeff_ptr += step; + iscan += step; + n_coeffs -= step; + + update_qp(qp); + + while (n_coeffs > 0) { + coeff = _mm256_loadu_si256((const __m256i *)coeff_ptr); + quantize(qp, &coeff, iscan, qcoeff_ptr, dqcoeff_ptr, &eob); + + coeff_ptr += step; + qcoeff_ptr += step; + dqcoeff_ptr += step; + iscan += step; + n_coeffs -= step; + } + { + __m256i eob_s; + eob_s = _mm256_shuffle_epi32(eob, 0xe); + eob = _mm256_max_epi16(eob, eob_s); + eob_s = _mm256_shufflelo_epi16(eob, 0xe); + eob = _mm256_max_epi16(eob, eob_s); + eob_s = _mm256_shufflelo_epi16(eob, 1); + eob = _mm256_max_epi16(eob, eob_s); + const __m128i final_eob = _mm_max_epi16(_mm256_castsi256_si128(eob), + _mm256_extractf128_si256(eob, 1)); + *eob_ptr = _mm_extract_epi16(final_eob, 0); + } + } else { + do { + const __m256i zero = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i *)qcoeff_ptr, zero); + _mm256_storeu_si256((__m256i *)dqcoeff_ptr, zero); + qcoeff_ptr += step; + dqcoeff_ptr += step; + n_coeffs -= step; + } while (n_coeffs > 0); + *eob_ptr = 0; + } +}
diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c index 84d4bbf..0d6ea01 100644 --- a/av1/encoder/av1_quantize.c +++ b/av1/encoder/av1_quantize.c
@@ -899,14 +899,29 @@ switch (qparam->log_scale) { case 0: - aom_highbd_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr, - pd->dequant, eob_ptr, sc->scan, sc->iscan + if (LIKELY(n_coeffs >= 8)) { + aom_highbd_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round, p->quant, p->quant_shift, qcoeff_ptr, + dqcoeff_ptr, pd->dequant, eob_ptr, sc->scan, + sc->iscan #if CONFIG_AOM_QM - , - qm_ptr, iqm_ptr + , + qm_ptr, iqm_ptr #endif - ); + ); + } else { + // TODO(luoyi): Need SIMD (e.g. sse2) for smaller block size + // quantization + aom_highbd_quantize_b_c(coeff_ptr, n_coeffs, skip_block, p->zbin, + p->round, p->quant, p->quant_shift, qcoeff_ptr, + dqcoeff_ptr, pd->dequant, eob_ptr, sc->scan, + sc->iscan +#if CONFIG_AOM_QM + , + qm_ptr, iqm_ptr +#endif + ); + } break; case 1: aom_highbd_quantize_b_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin,
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc index c522f9e..2d17163 100644 --- a/test/quantize_func_test.cc +++ b/test/quantize_func_test.cc
@@ -12,6 +12,7 @@ #include "third_party/googletest/src/googletest/include/gtest/gtest.h" #include "./aom_config.h" +#include "./aom_dsp_rtcd.h" #include "./av1_rtcd.h" #include "aom/aom_codec.h" #include "aom_ports/aom_timer.h" @@ -59,7 +60,10 @@ HBD_QUAN_FUNC; } -typedef std::tr1::tuple<QuantizeFunc, QuantizeFunc, TX_SIZE, aom_bit_depth_t> +typedef enum { TYPE_B, TYPE_DC, TYPE_FP } QuantType; + +typedef std::tr1::tuple<QuantizeFunc, QuantizeFunc, TX_SIZE, QuantType, + aom_bit_depth_t> QuantizeParam; typedef struct { @@ -73,7 +77,7 @@ protected: QuantizeTest() : quant_ref_(GET_PARAM(0)), quant_(GET_PARAM(1)), tx_size_(GET_PARAM(2)), - bd_(GET_PARAM(3)) {} + type_(GET_PARAM(3)), bd_(GET_PARAM(4)) {} virtual ~QuantizeTest() {} @@ -114,8 +118,17 @@ // Testing uses luminance quantization table const int16_t *zbin = qtab_->quant.y_zbin[q]; - const int16_t *round_fp = qtab_->quant.y_round_fp[q]; - const int16_t *quant_fp = qtab_->quant.y_quant_fp[q]; + + const int16_t *round = 0; + const int16_t *quant = 0; + if (type_ == TYPE_B) { + round = qtab_->quant.y_round[q]; + quant = qtab_->quant.y_quant[q]; + } else if (type_ == TYPE_FP) { + round = qtab_->quant.y_round_fp[q]; + quant = qtab_->quant.y_quant_fp[q]; + } + const int16_t *quant_shift = qtab_->quant.y_quant_shift[q]; const int16_t *dequant = qtab_->dequant.y_dequant[q]; @@ -124,13 +137,13 @@ memset(qcoeff_ref, 0, 5 * n_coeffs * sizeof(*qcoeff_ref)); - quant_ref_(coeff_ptr, n_coeffs, skip_block, zbin, round_fp, quant_fp, + quant_ref_(coeff_ptr, n_coeffs, skip_block, zbin, round, quant, quant_shift, qcoeff_ref, dqcoeff_ref, dequant, &eob[0], sc->scan, sc->iscan); ASM_REGISTER_STATE_CHECK(quant_( - coeff_ptr, n_coeffs, skip_block, zbin, round_fp, quant_fp, - quant_shift, qcoeff, dqcoeff, dequant, &eob[1], sc->scan, sc->iscan)); + coeff_ptr, n_coeffs, skip_block, zbin, round, quant, quant_shift, + qcoeff, dqcoeff, dequant, &eob[1], sc->scan, sc->iscan)); for (int j = 0; j < n_coeffs; ++j) { ASSERT_EQ(qcoeff_ref[j], qcoeff[j]) @@ -215,6 +228,7 @@ QuantizeFunc quant_ref_; QuantizeFunc quant_; TX_SIZE tx_size_; + QuantType type_; aom_bit_depth_t bd_; }; @@ -283,39 +297,46 @@ #if HAVE_AVX2 const QuantizeParam kQParamArrayAvx2[] = { - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X16, AOM_BITS_8), - make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32, + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X16, TYPE_FP, AOM_BITS_8), + make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32, + TYPE_FP, AOM_BITS_8), #if CONFIG_HIGHBITDEPTH make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_avx2>, TX_16X16, - AOM_BITS_8), + TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_avx2>, TX_16X16, - AOM_BITS_10), + TYPE_FP, AOM_BITS_10), make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_avx2>, TX_16X16, - AOM_BITS_12), + TYPE_FP, AOM_BITS_12), make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_avx2>, TX_32X32, - AOM_BITS_8), + TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_avx2>, TX_32X32, - AOM_BITS_10), + TYPE_FP, AOM_BITS_10), make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_avx2>, TX_32X32, - AOM_BITS_12), + TYPE_FP, AOM_BITS_12), #if CONFIG_TX64X64 make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_avx2>, TX_64X64, - AOM_BITS_8), + TYPE_FP, AOM_BITS_8), make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_avx2>, TX_64X64, - AOM_BITS_10), + TYPE_FP, AOM_BITS_10), make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>, &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_avx2>, TX_64X64, - AOM_BITS_12), + TYPE_FP, AOM_BITS_12), #endif // CONFIG_TX64X64 + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, TX_16X16, + TYPE_B, AOM_BITS_8), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, TX_16X16, + TYPE_B, AOM_BITS_10), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2, TX_16X16, + TYPE_B, AOM_BITS_12), #endif // CONFIG_HIGHBITDEPTH }; @@ -324,8 +345,24 @@ #endif // HAVE_AVX2 #if HAVE_SSE2 -const QuantizeParam kQParamArraySSE2[] = { make_tuple( - &av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_16X16, AOM_BITS_8) }; +const QuantizeParam kQParamArraySSE2[] = { + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_16X16, TYPE_FP, + AOM_BITS_8), +#if CONFIG_HIGHBITDEPTH + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, TX_16X16, + TYPE_B, AOM_BITS_8), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, TX_16X16, + TYPE_B, AOM_BITS_10), + make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_sse2, TX_16X16, + TYPE_B, AOM_BITS_12), + make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_sse2, + TX_32X32, TYPE_B, AOM_BITS_8), + make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_sse2, + TX_32X32, TYPE_B, AOM_BITS_10), + make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_sse2, + TX_32X32, TYPE_B, AOM_BITS_12), +#endif +}; INSTANTIATE_TEST_CASE_P(SSE2, QuantizeTest, ::testing::ValuesIn(kQParamArraySSE2)); @@ -333,7 +370,8 @@ #if !CONFIG_HIGHBITDEPTH && HAVE_SSSE3 && ARCH_X86_64 const QuantizeParam kQ16x16ParamArraySSSE3[] = { - make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_ssse3, TX_16X16, AOM_BITS_8), + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_ssse3, TX_16X16, TYPE_FP, + AOM_BITS_8), }; INSTANTIATE_TEST_CASE_P(SSSE3, QuantizeTest, ::testing::ValuesIn(kQ16x16ParamArraySSSE3)); @@ -341,7 +379,7 @@ // TODO(any): // The following test does not pass yet const QuantizeParam kQ32x32ParamArraySSSE3[] = { make_tuple( - av1_quantize_fp_32x32_c, av1_quantize_fp_32x32_ssse3, TX_32X32, + av1_quantize_fp_32x32_c, av1_quantize_fp_32x32_ssse3, TX_32X32, TYPE_FP, AOM_BITS_8) }; INSTANTIATE_TEST_CASE_P(DISABLED_SSSE3, QuantizeTest, ::testing::ValuesIn(kQ32x32ParamArraySSSE3));