Fix a bug when combining new-quant + supertx
Previously, we assumed that av1_init_plane_quantizers is always called with
segment_id == xd->mi[0]->mbmi.segment_id (and use the latter to derive the value
of 'qindex' to use in the quantizer). But this is no longer true when supertx
is enabled. This patch instead remembers the value of 'qindex' derived from
the latest call to av1_init_plane_quantizers and uses that directly.
Change-Id: Ifa1c5bf74cad29942ff79b88ca92c231bc07f336
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 8914ba5..7b6b431 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -95,8 +95,7 @@
#endif
const int shift = get_tx_scale(xd, tx_type, tx_size);
#if CONFIG_NEW_QUANT
- int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
- ref, plane_type);
+ int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq];
#else
const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift };
@@ -123,8 +122,7 @@
int shortcut = 0;
int next_shortcut = 0;
- assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
- (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
+ assert((mb->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
token_costs += band;
@@ -518,8 +516,7 @@
tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
- int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
- is_inter, plane_type);
+ int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
uint16_t *const eob = &p->eobs[block];
const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
const int16_t *src_diff;
@@ -527,8 +524,7 @@
FWD_TXFM_PARAM fwd_txfm_param;
- assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
- (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
+ assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
@@ -588,8 +584,7 @@
PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, is_inter);
- int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
- is_inter, plane_type);
+ int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
@@ -600,8 +595,7 @@
FWD_TXFM_PARAM fwd_txfm_param;
- assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
- (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
+ assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
@@ -665,13 +659,11 @@
const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
const int16_t *src_diff;
const int is_inter = is_inter_block(&xd->mi[0]->mbmi);
- int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
- is_inter, plane_type);
+ int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
FWD_TXFM_PARAM fwd_txfm_param;
- assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
- (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
+ assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
@@ -730,13 +722,11 @@
const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
const int16_t *src_diff;
const int is_inter = is_inter_block(&xd->mi[0]->mbmi);
- int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
- is_inter, plane_type);
+ int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
FWD_TXFM_PARAM fwd_txfm_param;
- assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
- (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
+ assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
