Rework get_eob_ctx to support rectangular tx_size Explicilty use the transform block size to determine the coeff band array. Remove the assumption on square transform block size. Change-Id: I18e285130465a5eced49304a27a6cb617e297760
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h index 9286b88..cbd0adf 100644 --- a/av1/common/txb_common.h +++ b/av1/common/txb_common.h
@@ -338,12 +338,12 @@ static INLINE int get_eob_ctx(const tran_low_t *tcoeffs, const int coeff_idx, // raster order - const int bwl) { + const TX_SIZE txs_ctx) { (void)tcoeffs; - if (bwl == 2) return av1_coeff_band_4x4[coeff_idx]; - if (bwl == 3) return av1_coeff_band_8x8[coeff_idx]; - if (bwl == 4) return av1_coeff_band_16x16[coeff_idx]; - if (bwl == 5) return av1_coeff_band_32x32[coeff_idx]; + if (txs_ctx == TX_4X4) return av1_coeff_band_4x4[coeff_idx]; + if (txs_ctx == TX_8X8) return av1_coeff_band_8x8[coeff_idx]; + if (txs_ctx == TX_16X16) return av1_coeff_band_16x16[coeff_idx]; + if (txs_ctx == TX_32X32) return av1_coeff_band_32x32[coeff_idx]; assert(0); return 0;
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c index 66e986b..74693b5 100644 --- a/av1/decoder/decodetxb.c +++ b/av1/decoder/decodetxb.c
@@ -88,7 +88,7 @@ for (c = 0; c < seg_eob; ++c) { int is_nz; int coeff_ctx = get_nz_map_ctx(tcoeffs, txb_mask, scan[c], bwl); - int eob_ctx = get_eob_ctx(tcoeffs, scan[c], bwl); + int eob_ctx = get_eob_ctx(tcoeffs, scan[c], txs_ctx); if (c < seg_eob - 1) is_nz = aom_read(r, nz_map[coeff_ctx], tx_size);
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index c2a99fb..acd915f 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -467,7 +467,6 @@ assert((mb->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0)); if (eob == 0) return eob; if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return eob; - #if CONFIG_PVQ (void)cm; (void)tx_size;
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index a40731e..bb334fa 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -102,7 +102,7 @@ for (c = 0; c < eob; ++c) { int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl); - int eob_ctx = get_eob_ctx(tcoeff, scan[c], bwl); + int eob_ctx = get_eob_ctx(tcoeff, scan[c], txs_ctx); tran_low_t v = tcoeff[scan[c]]; is_nz = (v != 0); @@ -396,7 +396,7 @@ } if (c < seg_eob) { - int eob_ctx = get_eob_ctx(qcoeff, scan[c], bwl); + int eob_ctx = get_eob_ctx(qcoeff, scan[c], txs_ctx); cost += av1_cost_bit(xd->fc->eob_flag[txs_ctx][plane_type][eob_ctx], c == (eob - 1)); } @@ -788,7 +788,7 @@ if (scan_idx < txb_info->seg_eob) { const int eob_ctx = - get_eob_ctx(txb_info->qcoeff, coeff_idx, txb_info->bwl); + get_eob_ctx(txb_info->qcoeff, coeff_idx, txb_info->txs_ctx); cost_diff -= av1_cost_bit(txb_probs->eob_flag[eob_ctx], scan_idx == (txb_info->eob - 1)); } @@ -860,6 +860,7 @@ const int nb_row = row - sig_ref_offset[i][0]; const int nb_col = col - sig_ref_offset[i][1]; const int nb_coeff_idx = nb_row * txb_info->stride + nb_col; + const int nb_scan_idx = iscan[nb_coeff_idx]; if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 && nb_row < txb_info->stride && nb_col < txb_info->stride) { @@ -932,7 +933,7 @@ cost += get_base_cost(abs_qc, ctx, txb_probs->coeff_base, base_idx); if (scan_idx < txb_info->seg_eob) { const int eob_ctx = - get_eob_ctx(txb_info->qcoeff, coeff_idx, txb_info->bwl); + get_eob_ctx(txb_info->qcoeff, coeff_idx, txb_info->txs_ctx); cost += av1_cost_bit(txb_probs->eob_flag[eob_ctx], scan_idx == (txb_info->eob - 1)); } @@ -989,7 +990,7 @@ // Note that get_eob_ctx does NOT actually account for qcoeff, so we don't // need to lower down the qcoeff here const int eob_ctx = - get_eob_ctx(txb_info->qcoeff, scan[*new_eob - 1], txb_info->bwl); + get_eob_ctx(txb_info->qcoeff, scan[*new_eob - 1], txb_info->txs_ctx); cost_diff -= av1_cost_bit(txb_probs->eob_flag[eob_ctx], 0); cost_diff += av1_cost_bit(txb_probs->eob_flag[eob_ctx], 1); } else { @@ -1144,7 +1145,7 @@ if (scan_idx < txb_info->seg_eob) { int eob_ctx = - get_eob_ctx(txb_info->qcoeff, scan[scan_idx], txb_info->bwl); + get_eob_ctx(txb_info->qcoeff, scan[scan_idx], txb_info->txs_ctx); cost += av1_cost_bit(txb_probs->eob_flag[eob_ctx], scan_idx == (txb_info->eob - 1)); } @@ -1478,9 +1479,9 @@ (x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2; const int64_t rddiv = x->rddiv; - TxbInfo txb_info = { qcoeff, dqcoeff, tcoeff, dequant, shift, - tx_size, bwl, stride, eob, seg_eob, - scan_order, txb_ctx, rdmult, rddiv }; + TxbInfo txb_info = { qcoeff, dqcoeff, tcoeff, dequant, shift, + tx_size, txs_ctx, bwl, stride, eob, + seg_eob, scan_order, txb_ctx, rdmult, rddiv }; TxbCache txb_cache; gen_txb_cache(&txb_cache, &txb_info); @@ -1583,7 +1584,7 @@ tran_low_t v = qcoeff[scan[c]]; int is_nz = (v != 0); int coeff_ctx = get_nz_map_ctx(tcoeff, txb_mask, scan[c], bwl); - int eob_ctx = get_eob_ctx(tcoeff, scan[c], bwl); + int eob_ctx = get_eob_ctx(tcoeff, scan[c], txsize_ctx); if (c == seg_eob - 1) break;
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h index 836033a..66d86e5 100644 --- a/av1/encoder/encodetxb.h +++ b/av1/encoder/encodetxb.h
@@ -30,6 +30,7 @@ const int16_t *dequant; int shift; TX_SIZE tx_size; + TX_SIZE txs_ctx; int bwl; int stride; int eob;