Use reduced_tx_set_used for all av1_get_tx_type calls
BUG=aomedia:1232
Change-Id: Ib2578cdb41128997f7513bc11f86edf43a799959
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 3f78182..41e2142 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1031,7 +1031,8 @@
// Only y plane's tx_type is transmitted
if (plane > 0) return;
PLANE_TYPE plane_type = get_plane_type(plane);
- TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
#endif
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index d7b7750..dd86dc2 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4528,8 +4528,8 @@
#else
// Only y plane's tx_type is updated
if (plane > 0) return;
- TX_TYPE tx_type =
- av1_get_tx_type(PLANE_TYPE_Y, xd, blk_row, blk_col, tx_size);
+ TX_TYPE tx_type = av1_get_tx_type(PLANE_TYPE_Y, xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
#endif
if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 &&
cm->base_qindex > 0 && !mbmi->skip &&
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 474839a..13478c2 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -134,8 +134,8 @@
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
const int16_t *const dequant_ptr = p->dequant_QTX;
const uint8_t *const band_translate = get_band_translate(tx_size);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order =
get_scan(cm, tx_size, tx_type, &xd->mi[0]->mbmi);
const int16_t *const scan = scan_order->scan;
@@ -446,7 +446,8 @@
const struct macroblock_plane *const p = &x->plane[plane];
const struct macroblockd_plane *const pd = &xd->plane[plane];
PLANE_TYPE plane_type = get_plane_type(plane);
- TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
#if CONFIG_NEW_QUANT
const int is_inter = is_inter_block(mbmi);
@@ -565,8 +566,8 @@
if (p->eobs[block]) {
*(args->skip) = 0;
- TX_TYPE tx_type =
- av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col, tx_size);
+ TX_TYPE tx_type = av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst,
pd->dst.stride, p->eobs[block],
cm->reduced_tx_set_used);
@@ -818,8 +819,8 @@
struct macroblockd_plane *const pd = &xd->plane[plane];
tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
PLANE_TYPE plane_type = get_plane_type(plane);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
uint16_t *eob = &p->eobs[block];
const int dst_stride = pd->dst.stride;
uint8_t *dst =
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 8d2323b..89ebcb0 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -444,8 +444,8 @@
MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *const scan = scan_order->scan;
const int seg_eob = av1_get_max_eob(tx_size);
@@ -684,8 +684,8 @@
const MACROBLOCKD *const xd = &x->e_mbd;
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
const PLANE_TYPE plane_type = get_plane_type(plane);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const struct macroblock_plane *p = &x->plane[plane];
const int eob = p->eobs[block];
@@ -2031,8 +2031,8 @@
MACROBLOCKD *const xd = &x->e_mbd;
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const struct macroblock_plane *p = &x->plane[plane];
struct macroblockd_plane *pd = &xd->plane[plane];
@@ -2148,8 +2148,8 @@
const uint16_t eob = p->eobs[block];
const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
const PLANE_TYPE plane_type = pd->plane_type;
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
(void)plane_bsize;
@@ -2174,8 +2174,8 @@
const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
const int segment_id = mbmi->segment_id;
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *const scan = scan_order->scan;
const int seg_eob = av1_get_tx_eob(&cpi->common.seg, segment_id, tx_size);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0e66128..a6bb904 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1660,8 +1660,8 @@
tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE tx_size = av1_get_tx_size(plane, xd);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, reduced_tx_set);
const int dst_stride = pd->dst.stride;
uint8_t *dst =
&pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]];
@@ -1764,7 +1764,8 @@
const PLANE_TYPE plane_type = get_plane_type(plane);
TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
+ cpi->common.reduced_tx_set_used);
av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, recon,
MAX_TX_SIZE, eob,
cpi->common.reduced_tx_set_used);
@@ -1827,7 +1828,8 @@
if (plane == 0)
mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type;
TX_TYPE ref_tx_type =
- av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size);
+ av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
if (tx_type != ref_tx_type) {
// use av1_get_tx_type() to check if the tx_type is valid for the current
// mode if it's not, we skip it here.
@@ -1946,8 +1948,8 @@
#if !CONFIG_TXK_SEL
const PLANE_TYPE plane_type = get_plane_type(plane);
- const TX_TYPE tx_type =
- av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
+ tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
int rate_cost = 0;
@@ -3612,7 +3614,8 @@
int64_t tmp;
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
PLANE_TYPE plane_type = get_plane_type(plane);
- TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
+ TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order =
get_scan(cm, tx_size, tx_type, &xd->mi[0]->mbmi);
BLOCK_SIZE txm_bsize = txsize_to_bsize[tx_size];
@@ -4653,7 +4656,8 @@
// Uses simple features on top of DCT coefficients to quickly predict
// whether optimal RD decision is to skip encoding the residual.
// The sse value is stored in dist.
-static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist) {
+static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist,
+ int reduced_tx_set) {
int max_tx_size =
get_max_rect_tx_size(bsize, is_inter_block(&x->e_mbd.mi[0]->mbmi));
if (tx_size_high[max_tx_size] > 16 || tx_size_wide[max_tx_size] > 16)
@@ -4684,11 +4688,9 @@
const struct macroblockd_plane *const pd = &xd->plane[0];
const BLOCK_SIZE plane_bsize =
get_plane_block_size(xd->mi[0]->mbmi.sb_type, pd);
- // TODO(sarahparker) This assumes reduced_tx_set_used == 0. I will do a
- // follow up refactor to make the actual value of reduced_tx_set_used
- // within this function.
- param.tx_set_type = get_ext_tx_set_type(param.tx_size, plane_bsize,
- is_inter_block(&xd->mi[0]->mbmi), 0);
+ param.tx_set_type =
+ get_ext_tx_set_type(param.tx_size, plane_bsize,
+ is_inter_block(&xd->mi[0]->mbmi), reduced_tx_set);
const uint32_t ac_q = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd);
uint32_t max_quantized_coef = 0;
const int bd_idx = (xd->bd == 8) ? 0 : ((xd->bd == 10) ? 1 : 2);
@@ -4831,7 +4833,7 @@
// context and terminate early.
int64_t dist;
if (is_inter && cpi->sf.tx_type_search.use_skip_flag_prediction &&
- predict_skip_flag(x, bsize, &dist)) {
+ predict_skip_flag(x, bsize, &dist, cm->reduced_tx_set_used)) {
set_skip_flag(cpi, x, rd_stats, bsize, dist);
// Save the RD search results into tx_rd_record.
if (within_border) save_tx_rd_info(n4, hash, x, rd_stats, tx_rd_record);
diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c
index d60f965..ad0eddb 100644
--- a/av1/encoder/tokenize.c
+++ b/av1/encoder/tokenize.c
@@ -240,7 +240,8 @@
struct macroblock_plane *p = &x->plane[plane];
struct macroblockd_plane *pd = &xd->plane[plane];
const PLANE_TYPE type = pd->plane_type;
- const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int rate = av1_cost_coeffs(
cpi, x, plane, blk_row, blk_col, block, tx_size, scan_order,
@@ -400,7 +401,8 @@
const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
const int segment_id = mbmi->segment_id;
const int16_t *scan, *nb;
- const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size);
+ const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size,
+ cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int ref = is_inter_block(mbmi);
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
