Use reduced_tx_set_used for all av1_get_tx_type calls BUG=aomedia:1232 Change-Id: Ib2578cdb41128997f7513bc11f86edf43a799959
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index c864938..304867f 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -916,13 +916,14 @@ static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd, int blk_row, - int blk_col, TX_SIZE tx_size) { + int blk_col, TX_SIZE tx_size, + int reduced_tx_set) { const MODE_INFO *const mi = xd->mi[0]; const MB_MODE_INFO *const mbmi = &mi->mbmi; const struct macroblockd_plane *const pd = &xd->plane[plane_type]; const BLOCK_SIZE plane_bsize = get_plane_block_size(mbmi->sb_type, pd); - const TxSetType tx_set_type = - get_ext_tx_set_type(tx_size, plane_bsize, is_inter_block(mbmi), 0); + const TxSetType tx_set_type = get_ext_tx_set_type( + tx_size, plane_bsize, is_inter_block(mbmi), reduced_tx_set); #if CONFIG_TXK_SEL TX_TYPE tx_type;
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index e97d018..b24949a 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -156,9 +156,11 @@ av1_read_coeffs_txb_facade(cm, xd, r, row, col, plane, tx_size, &max_scan_line, &eob); // tx_type will be read out in av1_read_coeffs_txb_facade - const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size, + cm->reduced_tx_set_used); #else // CONFIG_LV_MAP - const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size, + cm->reduced_tx_set_used); const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi); int16_t max_scan_line = 0; const int eob = @@ -218,11 +220,11 @@ av1_read_coeffs_txb_facade(cm, xd, r, blk_row, blk_col, plane, tx_size, &max_scan_line, &eob); // tx_type will be read out in av1_read_coeffs_txb_facade - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); #else // CONFIG_LV_MAP - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *sc = get_scan(cm, tx_size, tx_type, mbmi); int16_t max_scan_line = 0; const int eob =
diff --git a/av1/decoder/decodetxb.c b/av1/decoder/decodetxb.c index aea54cc..2bd4cb8 100644 --- a/av1/decoder/decodetxb.c +++ b/av1/decoder/decodetxb.c
@@ -122,8 +122,8 @@ #if CONFIG_TXK_SEL av1_read_tx_type(cm, xd, blk_row, blk_col, plane, tx_size, r); #endif - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const int16_t *const scan = scan_order->scan; int dummy;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 3f78182..41e2142 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -1031,7 +1031,8 @@ // Only y plane's tx_type is transmitted if (plane > 0) return; PLANE_TYPE plane_type = get_plane_type(plane); - TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); #endif const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index d7b7750..dd86dc2 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -4528,8 +4528,8 @@ #else // Only y plane's tx_type is updated if (plane > 0) return; - TX_TYPE tx_type = - av1_get_tx_type(PLANE_TYPE_Y, xd, blk_row, blk_col, tx_size); + TX_TYPE tx_type = av1_get_tx_type(PLANE_TYPE_Y, xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); #endif if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 && cm->base_qindex > 0 && !mbmi->skip &&
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index 474839a..13478c2 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -134,8 +134,8 @@ tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); const int16_t *const dequant_ptr = p->dequant_QTX; const uint8_t *const band_translate = get_band_translate(tx_size); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, &xd->mi[0]->mbmi); const int16_t *const scan = scan_order->scan; @@ -446,7 +446,8 @@ const struct macroblock_plane *const p = &x->plane[plane]; const struct macroblockd_plane *const pd = &xd->plane[plane]; PLANE_TYPE plane_type = get_plane_type(plane); - TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); #if CONFIG_NEW_QUANT const int is_inter = is_inter_block(mbmi); @@ -565,8 +566,8 @@ if (p->eobs[block]) { *(args->skip) = 0; - TX_TYPE tx_type = - av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col, tx_size); + TX_TYPE tx_type = av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst, pd->dst.stride, p->eobs[block], cm->reduced_tx_set_used); @@ -818,8 +819,8 @@ struct macroblockd_plane *const pd = &xd->plane[plane]; tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); PLANE_TYPE plane_type = get_plane_type(plane); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); uint16_t *eob = &p->eobs[block]; const int dst_stride = pd->dst.stride; uint8_t *dst =
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index 8d2323b..89ebcb0 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -444,8 +444,8 @@ MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; const PLANE_TYPE plane_type = get_plane_type(plane); const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const int16_t *const scan = scan_order->scan; const int seg_eob = av1_get_max_eob(tx_size); @@ -684,8 +684,8 @@ const MACROBLOCKD *const xd = &x->e_mbd; const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); const PLANE_TYPE plane_type = get_plane_type(plane); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; const struct macroblock_plane *p = &x->plane[plane]; const int eob = p->eobs[block]; @@ -2031,8 +2031,8 @@ MACROBLOCKD *const xd = &x->e_mbd; const PLANE_TYPE plane_type = get_plane_type(plane); const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; const struct macroblock_plane *p = &x->plane[plane]; struct macroblockd_plane *pd = &xd->plane[plane]; @@ -2148,8 +2148,8 @@ const uint16_t eob = p->eobs[block]; const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block); const PLANE_TYPE plane_type = pd->plane_type; - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); (void)plane_bsize; @@ -2174,8 +2174,8 @@ const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block); tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block); const int segment_id = mbmi->segment_id; - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const int16_t *const scan = scan_order->scan; const int seg_eob = av1_get_tx_eob(&cpi->common.seg, segment_id, tx_size);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 0e66128..a6bb904 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -1660,8 +1660,8 @@ tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); const PLANE_TYPE plane_type = get_plane_type(plane); const TX_SIZE tx_size = av1_get_tx_size(plane, xd); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, reduced_tx_set); const int dst_stride = pd->dst.stride; uint8_t *dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]]; @@ -1764,7 +1764,8 @@ const PLANE_TYPE plane_type = get_plane_type(plane); TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size, + cpi->common.reduced_tx_set_used); av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, recon, MAX_TX_SIZE, eob, cpi->common.reduced_tx_set_used); @@ -1827,7 +1828,8 @@ if (plane == 0) mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type; TX_TYPE ref_tx_type = - av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size); + av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); if (tx_type != ref_tx_type) { // use av1_get_tx_type() to check if the tx_type is valid for the current // mode if it's not, we skip it here. @@ -1946,8 +1948,8 @@ #if !CONFIG_TXK_SEL const PLANE_TYPE plane_type = get_plane_type(plane); - const TX_TYPE tx_type = - av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, + tx_size, cm->reduced_tx_set_used); const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi); int rate_cost = 0; @@ -3612,7 +3614,8 @@ int64_t tmp; tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block); PLANE_TYPE plane_type = get_plane_type(plane); - TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size); + TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, &xd->mi[0]->mbmi); BLOCK_SIZE txm_bsize = txsize_to_bsize[tx_size]; @@ -4653,7 +4656,8 @@ // Uses simple features on top of DCT coefficients to quickly predict // whether optimal RD decision is to skip encoding the residual. // The sse value is stored in dist. -static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist) { +static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist, + int reduced_tx_set) { int max_tx_size = get_max_rect_tx_size(bsize, is_inter_block(&x->e_mbd.mi[0]->mbmi)); if (tx_size_high[max_tx_size] > 16 || tx_size_wide[max_tx_size] > 16) @@ -4684,11 +4688,9 @@ const struct macroblockd_plane *const pd = &xd->plane[0]; const BLOCK_SIZE plane_bsize = get_plane_block_size(xd->mi[0]->mbmi.sb_type, pd); - // TODO(sarahparker) This assumes reduced_tx_set_used == 0. I will do a - // follow up refactor to make the actual value of reduced_tx_set_used - // within this function. - param.tx_set_type = get_ext_tx_set_type(param.tx_size, plane_bsize, - is_inter_block(&xd->mi[0]->mbmi), 0); + param.tx_set_type = + get_ext_tx_set_type(param.tx_size, plane_bsize, + is_inter_block(&xd->mi[0]->mbmi), reduced_tx_set); const uint32_t ac_q = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd); uint32_t max_quantized_coef = 0; const int bd_idx = (xd->bd == 8) ? 0 : ((xd->bd == 10) ? 1 : 2); @@ -4831,7 +4833,7 @@ // context and terminate early. int64_t dist; if (is_inter && cpi->sf.tx_type_search.use_skip_flag_prediction && - predict_skip_flag(x, bsize, &dist)) { + predict_skip_flag(x, bsize, &dist, cm->reduced_tx_set_used)) { set_skip_flag(cpi, x, rd_stats, bsize, dist); // Save the RD search results into tx_rd_record. if (within_border) save_tx_rd_info(n4, hash, x, rd_stats, tx_rd_record);
diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c index d60f965..ad0eddb 100644 --- a/av1/encoder/tokenize.c +++ b/av1/encoder/tokenize.c
@@ -240,7 +240,8 @@ struct macroblock_plane *p = &x->plane[plane]; struct macroblockd_plane *pd = &xd->plane[plane]; const PLANE_TYPE type = pd->plane_type; - const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const int rate = av1_cost_coeffs( cpi, x, plane, blk_row, blk_col, block, tx_size, scan_order, @@ -400,7 +401,8 @@ const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block); const int segment_id = mbmi->segment_id; const int16_t *scan, *nb; - const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size); + const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size, + cm->reduced_tx_set_used); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const int ref = is_inter_block(mbmi); FRAME_CONTEXT *ec_ctx = xd->tile_ctx;