Remove bands from new-quant profiles Rather than having a set of parameters for each of the 6 COEF_BANDS, we have 1 for DC and 1 for AC coefficients. No change in performance since all of the bands had the same parameters. Change-Id: I3665e7c1b21f117be776f371d87d64b097715735
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index ea4fd32..3bf4f27 100755 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -157,18 +157,18 @@ } if (aom_config("CONFIG_NEW_QUANT") eq "yes") { - add_proto qw/void quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; if (aom_config("CONFIG_TX64X64") eq "yes") { - add_proto qw/void quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; } } @@ -445,18 +445,18 @@ # ENCODEMB INVOKE if (aom_config("CONFIG_NEW_QUANT") eq "yes") { - add_proto qw/void highbd_quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void highbd_quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void highbd_quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void highbd_quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void highbd_quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void highbd_quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void highbd_quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void highbd_quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; if (aom_config("CONFIG_TX64X64") eq "yes") { - add_proto qw/void highbd_quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void highbd_quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; - add_proto qw/void highbd_quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const uint8_t *band"; + add_proto qw/void highbd_quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan"; } }
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index e78ade1..3b11ee5 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -498,8 +498,7 @@ // shift/scale as TX. int16_t seg_dequant_QTX[MAX_SEGMENTS][2]; #if CONFIG_NEW_QUANT - dequant_val_type_nuq seg_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES] - [COEF_BANDS]; + dequant_val_type_nuq seg_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2]; #endif uint8_t *color_index_map;
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h index 3110240..e43f520 100644 --- a/av1/common/onyxc_int.h +++ b/av1/common/onyxc_int.h
@@ -352,12 +352,9 @@ int max_qmlevel; #endif #if CONFIG_NEW_QUANT - dequant_val_type_nuq y_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES] - [COEF_BANDS]; - dequant_val_type_nuq u_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES] - [COEF_BANDS]; - dequant_val_type_nuq v_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES] - [COEF_BANDS]; + dequant_val_type_nuq y_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2]; + dequant_val_type_nuq u_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2]; + dequant_val_type_nuq v_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2]; #endif /* We allocate a MODE_INFO struct for each macroblock, together with
diff --git a/av1/common/quant_common.c b/av1/common/quant_common.c index 5bec9a2..44a31ff 100644 --- a/av1/common/quant_common.c +++ b/av1/common/quant_common.c
@@ -31,54 +31,38 @@ uint8_t doff; // dequantization } qprofile_type; -static const qprofile_type nuq[QUANT_PROFILES][COEF_BANDS] = { +static const qprofile_type nuq[QUANT_PROFILES][2] = { { // lossless - { { 64, 128, 128 }, 0 }, // dc, band 0 - { { 64, 128, 128 }, 0 }, // band 1 - { { 64, 128, 128 }, 0 }, // band 2 - { { 64, 128, 128 }, 0 }, // band 3 - { { 64, 128, 128 }, 0 }, // band 4 - { { 64, 128, 128 }, 0 }, // band 5 + { { 64, 128, 128 }, 0 }, // dc + { { 64, 128, 128 }, 0 }, // ac }, { - { { 64, 128, 128 }, 4 }, // dc, band 0 - { { 64, 128, 128 }, 4 }, // band 1 - { { 64, 128, 128 }, 4 }, // band 2 - { { 64, 128, 128 }, 4 }, // band 3 - { { 64, 128, 128 }, 4 }, // band 4 - { { 64, 128, 128 }, 4 } // band 5 + { { 64, 128, 128 }, 4 }, // dc + { { 64, 128, 128 }, 4 }, // ac }, { - { { 64, 128, 128 }, 6 }, // dc, band 0 - { { 64, 128, 128 }, 6 }, // band 1 - { { 64, 128, 128 }, 6 }, // band 2 - { { 64, 128, 128 }, 6 }, // band 3 - { { 64, 128, 128 }, 6 }, // band 4 - { { 64, 128, 128 }, 6 } // band 5 + { { 64, 128, 128 }, 6 }, // dc + { { 64, 128, 128 }, 6 }, // ac }, { - { { 64, 128, 128 }, 8 }, // dc, band 0 - { { 64, 128, 128 }, 8 }, // band 1 - { { 64, 128, 128 }, 8 }, // band 2 - { { 64, 128, 128 }, 8 }, // band 3 - { { 64, 128, 128 }, 8 }, // band 4 - { { 64, 128, 128 }, 8 } // band 5 + { { 64, 128, 128 }, 8 }, // dc + { { 64, 128, 128 }, 8 }, // ac } }; -static const uint8_t *get_nuq_knots(int band, int q_profile) { - return nuq[q_profile][band].knots; +static const uint8_t *get_nuq_knots(int is_ac_coeff, int q_profile) { + return nuq[q_profile][is_ac_coeff].knots; } -static INLINE int16_t quant_to_doff_fixed(int band, int q_profile) { - return nuq[q_profile][band].doff; +static INLINE int16_t quant_to_doff_fixed(int is_ac_coeff, int q_profile) { + return nuq[q_profile][is_ac_coeff].doff; } // get cumulative bins -static INLINE void get_cuml_bins_nuq(int q, int band, tran_low_t *cuml_bins, - int q_profile) { - const uint8_t *knots = get_nuq_knots(band, q_profile); +static INLINE void get_cuml_bins_nuq(int q, int is_ac_coeff, + tran_low_t *cuml_bins, int q_profile) { + const uint8_t *knots = get_nuq_knots(is_ac_coeff, q_profile); int16_t cuml_knots[NUQ_KNOTS]; int i; cuml_knots[0] = knots[0]; @@ -87,22 +71,22 @@ cuml_bins[i] = ROUND_POWER_OF_TWO(cuml_knots[i] * q, 7); } -void av1_get_dequant_val_nuq(int q, int band, tran_low_t *dq, +void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq, tran_low_t *cuml_bins, int q_profile) { - const uint8_t *knots = get_nuq_knots(band, q_profile); + const uint8_t *knots = get_nuq_knots(is_ac_coeff, q_profile); tran_low_t cuml_bins_[NUQ_KNOTS], *cuml_bins_ptr; tran_low_t doff; int i; cuml_bins_ptr = (cuml_bins ? cuml_bins : cuml_bins_); - get_cuml_bins_nuq(q, band, cuml_bins_ptr, q_profile); + get_cuml_bins_nuq(q, is_ac_coeff, cuml_bins_ptr, q_profile); dq[0] = 0; for (i = 1; i < NUQ_KNOTS; ++i) { - doff = quant_to_doff_fixed(band, q_profile); + doff = quant_to_doff_fixed(is_ac_coeff, q_profile); doff = ROUND_POWER_OF_TWO(doff * knots[i], 7); dq[i] = cuml_bins_ptr[i - 1] + ROUND_POWER_OF_TWO((knots[i] - doff * 2) * q, 8); } - doff = quant_to_doff_fixed(band, q_profile); + doff = quant_to_doff_fixed(is_ac_coeff, q_profile); dq[NUQ_KNOTS] = cuml_bins_ptr[NUQ_KNOTS - 1] + ROUND_POWER_OF_TWO((64 - doff) * q, 7); }
diff --git a/av1/common/quant_common.h b/av1/common/quant_common.h index 49dfd59..a8d1114 100644 --- a/av1/common/quant_common.h +++ b/av1/common/quant_common.h
@@ -85,7 +85,7 @@ typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS + 1]; typedef tran_low_t cuml_bins_type_nuq[NUQ_KNOTS]; -void av1_get_dequant_val_nuq(int q, int band, tran_low_t *dq, +void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq, tran_low_t *cuml_bins, int dq_off_index); tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq); tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 5aa57fd..2a1a284 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -1303,7 +1303,8 @@ #endif // CONFIG_AOM_QM #if CONFIG_NEW_QUANT for (int dq = 0; dq < QUANT_PROFILES; dq++) { - for (int b = 0; b < COEF_BANDS; ++b) { + // DC and AC coefs + for (int b = 0; b < 2; ++b) { av1_get_dequant_val_nuq(cm->y_dequant_QTX[i][b != 0], b, cm->y_dequant_nuq_QTX[i][dq][b], NULL, dq); av1_get_dequant_val_nuq(cm->u_dequant_QTX[i][b != 0], b,
diff --git a/av1/decoder/detokenize.c b/av1/decoder/detokenize.c index 1ad1b10..40e8e21 100644 --- a/av1/decoder/detokenize.c +++ b/av1/decoder/detokenize.c
@@ -145,7 +145,7 @@ int first_pos = (c == 0); #if CONFIG_NEW_QUANT - dqv_val = &dq_val[band][0]; + dqv_val = &dq_val[band != 0][0]; #endif // CONFIG_NEW_QUANT comb_token = last_pos ? 2 * av1_read_record_bit(xd->counts, r, ACCT_STR) + 2 @@ -186,7 +186,7 @@ token += av1_read_record_symbol(xd->counts, r, coef_tail_cdfs[band][ctx], TAIL_TOKENS, ACCT_STR); #if CONFIG_NEW_QUANT - dqv_val = &dq_val[band][0]; + dqv_val = &dq_val[band != 0][0]; #endif // CONFIG_NEW_QUANT *max_scan_line = AOMMAX(*max_scan_line, scan[c]);
diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c index dde3011..18b226c 100644 --- a/av1/encoder/av1_quantize.c +++ b/av1/encoder/av1_quantize.c
@@ -280,8 +280,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -291,7 +290,7 @@ const int rc = scan[i]; if (quantize_coeff_nuq(coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], dequant_ptr[rc != 0], - cuml_bins_ptr[band[i]], dequant_val[band[i]], + cuml_bins_ptr[rc != 0], dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) eob = i; } @@ -305,8 +304,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -315,8 +313,8 @@ for (i = 0; i < n_coeffs; i++) { const int rc = scan[i]; if (quantize_coeff_fp_nuq(coeff_ptr[rc], quant_ptr[rc != 0], - dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], - dequant_val[band[i]], &qcoeff_ptr[rc], + dequant_ptr[rc != 0], cuml_bins_ptr[rc != 0], + dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) eob = i; } @@ -331,8 +329,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -342,8 +339,8 @@ const int rc = scan[i]; if (quantize_coeff_bigtx_nuq( coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], - dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], - dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + dequant_ptr[rc != 0], cuml_bins_ptr[rc != 0], + dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_32X32))) eob = i; } @@ -357,8 +354,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -368,7 +364,7 @@ const int rc = scan[i]; if (quantize_coeff_bigtx_fp_nuq( coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], - cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + cuml_bins_ptr[rc != 0], dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_32X32))) eob = i; } @@ -384,8 +380,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -395,8 +390,8 @@ const int rc = scan[i]; if (quantize_coeff_bigtx_nuq( coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], - dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], - dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + dequant_ptr[rc != 0], cuml_bins_ptr[rc != 0], + dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_64X64))) eob = i; } @@ -410,8 +405,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -421,7 +415,7 @@ const int rc = scan[i]; if (quantize_coeff_bigtx_fp_nuq( coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], - cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + cuml_bins_ptr[rc != 0], dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_64X64))) eob = i; } @@ -743,7 +737,6 @@ const QUANT_PARAM *qparam) { // obsolete skip_block const int skip_block = 0; - const uint8_t *band = get_band_translate(qparam->tx_size); int dq = qparam->dq; switch (qparam->log_scale) { @@ -752,14 +745,14 @@ p->quant_shift_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], - qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan); break; case 1: quantize_32x32_nuq( coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #if CONFIG_TX64X64 case 2: @@ -767,7 +760,7 @@ coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #endif // CONFIG_TX64X64 default: assert(0); @@ -781,7 +774,6 @@ const QUANT_PARAM *qparam) { // obsolete skip_block const int skip_block = 0; - const uint8_t *band = get_band_translate(qparam->tx_size); int dq = qparam->dq; switch (qparam->log_scale) { @@ -790,14 +782,14 @@ p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], - qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band); + qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan); break; case 1: quantize_32x32_fp_nuq( coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #if CONFIG_TX64X64 case 2: @@ -805,7 +797,7 @@ coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #endif // CONFIG_TX64X64 default: assert(0); @@ -1173,8 +1165,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -1184,23 +1175,20 @@ const int rc = scan[i]; if (highbd_quantize_coeff_nuq( coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], - dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], - dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) + dequant_ptr[rc != 0], cuml_bins_ptr[rc != 0], + dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) eob = i; } } *eob_ptr = eob + 1; } -void highbd_quantize_32x32_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *quant_ptr, - const int16_t *quant_shift_ptr, - const int16_t *dequant_ptr, - const cuml_bins_type_nuq *cuml_bins_ptr, - const dequant_val_type_nuq *dequant_val, - tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, - const int16_t *scan, const uint8_t *band) { +void highbd_quantize_32x32_nuq_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -1210,8 +1198,8 @@ const int rc = scan[i]; if (highbd_quantize_coeff_bigtx_nuq( coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], - dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], - dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + dequant_ptr[rc != 0], cuml_bins_ptr[rc != 0], + dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_32X32))) eob = i; } @@ -1219,15 +1207,12 @@ *eob_ptr = eob + 1; } -void highbd_quantize_32x32_fp_nuq_c(const tran_low_t *coeff_ptr, - intptr_t n_coeffs, int skip_block, - const int16_t *quant_ptr, - const int16_t *dequant_ptr, - const cuml_bins_type_nuq *cuml_bins_ptr, - const dequant_val_type_nuq *dequant_val, - tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, - const int16_t *scan, const uint8_t *band) { +void highbd_quantize_32x32_fp_nuq_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *quant_ptr, const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -1237,7 +1222,7 @@ const int rc = scan[i]; if (highbd_quantize_coeff_bigtx_fp_nuq( coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], - cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + cuml_bins_ptr[rc != 0], dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_32X32))) eob = i; } @@ -1246,15 +1231,12 @@ } #if CONFIG_TX64X64 -void highbd_quantize_64x64_nuq_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *quant_ptr, - const int16_t *quant_shift_ptr, - const int16_t *dequant_ptr, - const cuml_bins_type_nuq *cuml_bins_ptr, - const dequant_val_type_nuq *dequant_val, - tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, - const int16_t *scan, const uint8_t *band) { +void highbd_quantize_64x64_nuq_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, + const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -1264,8 +1246,8 @@ const int rc = scan[i]; if (highbd_quantize_coeff_bigtx_nuq( coeff_ptr[rc], quant_ptr[rc != 0], quant_shift_ptr[rc != 0], - dequant_ptr[rc != 0], cuml_bins_ptr[band[i]], - dequant_val[band[i]], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], + dequant_ptr[rc != 0], cuml_bins_ptr[rc != 0], + dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_64X64))) eob = i; } @@ -1273,15 +1255,12 @@ *eob_ptr = eob + 1; } -void highbd_quantize_64x64_fp_nuq_c(const tran_low_t *coeff_ptr, - intptr_t n_coeffs, int skip_block, - const int16_t *quant_ptr, - const int16_t *dequant_ptr, - const cuml_bins_type_nuq *cuml_bins_ptr, - const dequant_val_type_nuq *dequant_val, - tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, - const int16_t *scan, const uint8_t *band) { +void highbd_quantize_64x64_fp_nuq_c( + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, + const int16_t *quant_ptr, const int16_t *dequant_ptr, + const cuml_bins_type_nuq *cuml_bins_ptr, + const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, + tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -1291,7 +1270,7 @@ const int rc = scan[i]; if (highbd_quantize_coeff_bigtx_fp_nuq( coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], - cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + cuml_bins_ptr[rc != 0], dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc], av1_get_tx_scale(TX_64X64))) eob = i; } @@ -1306,8 +1285,7 @@ const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const uint8_t *band) { + uint16_t *eob_ptr, const int16_t *scan) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); @@ -1317,7 +1295,7 @@ const int rc = scan[i]; if (highbd_quantize_coeff_fp_nuq( coeff_ptr[rc], quant_ptr[rc != 0], dequant_ptr[rc != 0], - cuml_bins_ptr[band[i]], dequant_val[band[i]], &qcoeff_ptr[rc], + cuml_bins_ptr[rc != 0], dequant_val[rc != 0], &qcoeff_ptr[rc], &dqcoeff_ptr[rc])) eob = i; } @@ -1405,7 +1383,6 @@ const SCAN_ORDER *sc, const QUANT_PARAM *qparam) { // obsolete skip_block const int skip_block = 0; - const uint8_t *band = get_band_translate(qparam->tx_size); const int dq = qparam->dq; switch (qparam->log_scale) { @@ -1414,14 +1391,14 @@ coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; case 1: highbd_quantize_32x32_nuq( coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #if CONFIG_TX64X64 case 2: @@ -1429,7 +1406,7 @@ coeff_ptr, n_coeffs, skip_block, p->quant_QTX, p->quant_shift_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #endif // CONFIG_TX64X64 default: assert(0); @@ -1442,7 +1419,6 @@ const SCAN_ORDER *sc, const QUANT_PARAM *qparam) { // obsolete skip_block const int skip_block = 0; - const uint8_t *band = get_band_translate(qparam->tx_size); const int dq = qparam->dq; switch (qparam->log_scale) { @@ -1451,14 +1427,14 @@ coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; case 1: highbd_quantize_32x32_fp_nuq( coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #if CONFIG_TX64X64 case 2: @@ -1466,7 +1442,7 @@ coeff_ptr, n_coeffs, skip_block, p->quant_fp_QTX, p->dequant_QTX, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq], (const dequant_val_type_nuq *)p->dequant_val_nuq_QTX[dq], qcoeff_ptr, - dqcoeff_ptr, eob_ptr, sc->scan, band); + dqcoeff_ptr, eob_ptr, sc->scan); break; #endif // CONFIG_TX64X64 default: assert(0); @@ -1608,7 +1584,8 @@ #if CONFIG_NEW_QUANT int dq; for (dq = 0; dq < QUANT_PROFILES; dq++) { - for (i = 0; i < COEF_BANDS; i++) { + // DC and AC coefs + for (i = 0; i < 2; i++) { const int y_quant = deq->y_dequant_QTX[q][i != 0]; const int u_quant = deq->u_dequant_QTX[q][i != 0]; const int v_quant = deq->v_dequant_QTX[q][i != 0];
diff --git a/av1/encoder/av1_quantize.h b/av1/encoder/av1_quantize.h index bce33ef..94156b5 100644 --- a/av1/encoder/av1_quantize.h +++ b/av1/encoder/av1_quantize.h
@@ -45,15 +45,12 @@ // All of its fields use the same coefficient shift/scaling at TX. typedef struct { #if CONFIG_NEW_QUANT - DECLARE_ALIGNED( - 16, tran_low_t, - y_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]); - DECLARE_ALIGNED( - 16, tran_low_t, - u_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]); - DECLARE_ALIGNED( - 16, tran_low_t, - v_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS][NUQ_KNOTS]); + DECLARE_ALIGNED(16, tran_low_t, + y_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][2][NUQ_KNOTS]); + DECLARE_ALIGNED(16, tran_low_t, + u_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][2][NUQ_KNOTS]); + DECLARE_ALIGNED(16, tran_low_t, + v_cuml_bins_nuq[QUANT_PROFILES][QINDEX_RANGE][2][NUQ_KNOTS]); #endif // CONFIG_NEW_QUANT // 0: dc 1: ac 2-8: ac repeated to SIMD width DECLARE_ALIGNED(16, int16_t, y_quant[QINDEX_RANGE][8]); @@ -95,15 +92,12 @@ DECLARE_ALIGNED(16, int16_t, u_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width DECLARE_ALIGNED(16, int16_t, v_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width #if CONFIG_NEW_QUANT - DECLARE_ALIGNED( - 16, dequant_val_type_nuq, - y_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]); - DECLARE_ALIGNED( - 16, dequant_val_type_nuq, - u_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]); - DECLARE_ALIGNED( - 16, dequant_val_type_nuq, - v_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][COEF_BANDS]); + DECLARE_ALIGNED(16, dequant_val_type_nuq, + y_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]); + DECLARE_ALIGNED(16, dequant_val_type_nuq, + u_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]); + DECLARE_ALIGNED(16, dequant_val_type_nuq, + v_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]); #endif // CONFIG_NEW_QUANT } Dequants;
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index dc50064..8e7888f6 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -259,15 +259,14 @@ if (x_a != 0) { #if CONFIG_DAALA_TX #if CONFIG_NEW_QUANT - dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[band_translate[i]]) - - coeff[rc]; + dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0]) - coeff[rc]; #else // CONFIG_NEW_QUANT dx -= (dqv + sz) ^ sz; #endif // CONFIG_NEW_QUANT d2_a = ((int64_t)dx * dx + depth_round) >> depth_shift; #else // CONFIG_DAALA_TX #if CONFIG_NEW_QUANT - dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[band_translate[i]]) - + dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0]) - (coeff[rc] * (1 << shift)); dx >>= xd->bd - 8; #else // CONFIG_NEW_QUANT @@ -349,16 +348,14 @@ if (x_a != 0) { #if CONFIG_DAALA_TX #if CONFIG_NEW_QUANT - dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, - dequant_val[band_translate[i]]); + dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0]); if (sz) dqc_a = -dqc_a; #else dqc_a = x_a * dqv; #endif // CONFIG_NEW_QUANT #else // CONFIG_DAALA_TX #if CONFIG_NEW_QUANT - dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, - dequant_val[band_translate[i]]); + dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0]); dqc_a = shift ? ROUND_POWER_OF_TWO(dqc_a, shift) : dqc_a; if (sz) dqc_a = -dqc_a; #else