Replace RECT_TX_EXT experiment
Remove the previous experiment and now use the same name for a
simpler experiment that only enables 4:1 transforms for 4:1
partitions when ext_partition_types is on, and that which was
previously enabled with the USE_RECT_TX_EXT macro.
Change-Id: Iccc35744bd292abf3c187da6f23b787692d50296
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 2259946..83ab458 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -478,11 +478,7 @@
int visible_cols) {
unsigned sse;
- if (txb_rows == visible_rows && txb_cols == visible_cols
-#if CONFIG_RECT_TX_EXT
- && tx_bsize < BLOCK_SIZES
-#endif
- ) {
+ if (txb_rows == visible_rows && txb_cols == visible_cols) {
cpi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
return sse;
}
@@ -1807,9 +1803,7 @@
BLOCK_SIZE plane_bsize, int blk_row, int blk_col,
BLOCK_SIZE tx_bsize, int *width, int *height,
int *visible_width, int *visible_height) {
-#if !(CONFIG_RECT_TX_EXT)
assert(tx_bsize <= plane_bsize);
-#endif
int txb_height = block_size_high[tx_bsize];
int txb_width = block_size_wide[tx_bsize];
const int block_height = block_size_high[plane_bsize];
@@ -2381,11 +2375,6 @@
const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
const int tx_size_ctx = get_tx_size_context(xd);
int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
-#if CONFIG_RECT_TX_EXT
- if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
- r_tx_size +=
- x->quarter_tx_size_cost[tx_size == quarter_txsize_lookup[bsize]];
-#endif
return r_tx_size;
} else {
return 0;
@@ -2709,55 +2698,6 @@
}
}
-#if CONFIG_RECT_TX_EXT
- // test 1:4/4:1 tx
- int evaluate_quarter_tx = 0;
- if (is_quarter_tx_allowed(xd, mbmi, is_inter)) {
- if (tx_select) {
- evaluate_quarter_tx = 1;
- } else {
- const TX_SIZE chosen_tx_size =
- tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
- evaluate_quarter_tx = chosen_tx_size == quarter_txsize_lookup[bs];
- }
- }
- if (evaluate_quarter_tx) {
- TX_TYPE tx_start = DCT_DCT;
- TX_TYPE tx_end = TX_TYPES;
-#if CONFIG_TXK_SEL
- // The tx_type becomes dummy when lv_map is on. The tx_type search will be
- // performed in av1_search_txk_type()
- tx_end = DCT_DCT + 1;
-#endif
- TX_TYPE tx_type;
- for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
- if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
- const TX_SIZE tx_size = quarter_txsize_lookup[bs];
- RD_STATS this_rd_stats;
- const TxSetType tx_set_type =
- get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used);
- if (av1_ext_tx_used[tx_set_type][tx_type]) {
- rd =
- txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, tx_size);
- if (rd < best_rd) {
-#if CONFIG_TXK_SEL
- memcpy(best_txk_type, mbmi->txk_type,
- sizeof(best_txk_type[0]) * num_blk);
-#endif
- best_tx_type = tx_type;
- best_tx_size = tx_size;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- }
-#if !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
- const int is_inter = is_inter_block(mbmi);
- if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
-#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
- }
- }
-#endif // CONFIG_RECT_TX_EXT
-
if (tx_select) {
start_tx = max_tx_size;
end_tx = AOMMAX((int)TX_4X4, start_tx - MAX_TX_DEPTH + evaluate_rect_tx);
@@ -3921,24 +3861,6 @@
TX_TYPE best_tx_type = TX_TYPES;
int txk_idx = (blk_row << 4) + blk_col;
#endif
-#if CONFIG_RECT_TX_EXT
- TX_SIZE quarter_txsize = quarter_txsize_lookup[mbmi->sb_type];
- int check_qttx = is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) &&
- tx_size == get_max_rect_tx_size(mbmi->sb_type, 1) &&
- quarter_txsize != tx_size;
- int is_qttx_picked = 0;
- int eobs_qttx[2] = { 0, 0 };
- int skip_qttx[2] = { 0, 0 };
- int block_offset_qttx = check_qttx
- ? tx_size_wide_unit[quarter_txsize] *
- tx_size_high_unit[quarter_txsize]
- : 0;
- int blk_row_offset, blk_col_offset;
- int is_wide_qttx =
- tx_size_wide_unit[quarter_txsize] > tx_size_high_unit[quarter_txsize];
- blk_row_offset = is_wide_qttx ? tx_size_high_unit[quarter_txsize] : 0;
- blk_col_offset = is_wide_qttx ? 0 : tx_size_wide_unit[quarter_txsize];
-#endif
av1_init_rd_stats(&sum_rd_stats);
@@ -3998,12 +3920,6 @@
if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH)
rd_stats->rate += x->txfm_partition_cost[ctx][0];
-#if CONFIG_RECT_TX_EXT
- if (check_qttx) {
- assert(blk_row == 0 && blk_col == 0);
- rd_stats->rate += x->quarter_tx_size_cost[0];
- }
-#endif
this_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist);
#if CONFIG_LV_MAP
tmp_eob = p->txb_entropy_ctx[block];
@@ -4014,104 +3930,6 @@
#if CONFIG_TXK_SEL
best_tx_type = mbmi->txk_type[txk_idx];
#endif
-
-#if CONFIG_RECT_TX_EXT
- if (check_qttx) {
- assert(blk_row == 0 && blk_col == 0 && block == 0 && plane == 0);
-
- RD_STATS rd_stats_tmp, rd_stats_qttx;
- int64_t rd_qttx;
-
- av1_init_rd_stats(&rd_stats_qttx);
- av1_init_rd_stats(&rd_stats_tmp);
-
- av1_tx_block_rd_b(cpi, x, quarter_txsize, 0, 0, plane, 0, plane_bsize,
- pta, ptl, &rd_stats_qttx, fast);
- if (rd_stats->rate == INT_MAX) return;
-
- tx_size_ctx = txsize_sqr_map[quarter_txsize];
- coeff_ctx = get_entropy_context(quarter_txsize, pta, ptl);
- zero_blk_rate =
- x->token_head_costs[tx_size_ctx][pd->plane_type][1][0][coeff_ctx][0];
- if ((RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist) >=
- RDCOST(x->rdmult, zero_blk_rate, rd_stats_qttx.sse) ||
- rd_stats_qttx.skip == 1) &&
- !xd->lossless[mbmi->segment_id]) {
-#if CONFIG_RD_DEBUG
- av1_update_txb_coeff_cost(&rd_stats_qttx, plane, quarter_txsize, 0, 0,
- zero_blk_rate - rd_stats_qttx.rate);
-#endif // CONFIG_RD_DEBUG
- rd_stats_qttx.rate = zero_blk_rate;
- rd_stats_qttx.dist = rd_stats_qttx.sse;
- rd_stats_qttx.skip = 1;
- x->blk_skip[plane][blk_row * bw + blk_col] = 1;
- skip_qttx[0] = 1;
- p->eobs[block] = 0;
- } else {
- x->blk_skip[plane][blk_row * bw + blk_col] = 0;
- skip_qttx[0] = 0;
- rd_stats->skip = 0;
- }
-
- // Second tx block
- av1_tx_block_rd_b(cpi, x, quarter_txsize, blk_row_offset, blk_col_offset,
- plane, block_offset_qttx, plane_bsize, pta, ptl,
- &rd_stats_tmp, fast);
-
- if (rd_stats->rate == INT_MAX) return;
-
- av1_set_txb_context(x, plane, 0, quarter_txsize, pta, ptl);
-
- coeff_ctx = get_entropy_context(quarter_txsize, pta + blk_col_offset,
- ptl + blk_row_offset);
- zero_blk_rate =
- x->token_head_costs[tx_size_ctx][pd->plane_type][1][0][coeff_ctx][0];
- if ((RDCOST(x->rdmult, rd_stats_tmp.rate, rd_stats_tmp.dist) >=
- RDCOST(x->rdmult, zero_blk_rate, rd_stats_tmp.sse) ||
- rd_stats_tmp.skip == 1) &&
- !xd->lossless[mbmi->segment_id]) {
-#if CONFIG_RD_DEBUG
- av1_update_txb_coeff_cost(&rd_stats_tmp, plane, quarter_txsize, 0, 0,
- zero_blk_rate - rd_stats_tmp.rate);
-#endif // CONFIG_RD_DEBUG
- rd_stats_tmp.rate = zero_blk_rate;
- rd_stats_tmp.dist = rd_stats_tmp.sse;
- rd_stats_tmp.skip = 1;
- x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = 1;
- skip_qttx[1] = 1;
- p->eobs[block_offset_qttx] = 0;
- } else {
- x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = 0;
- skip_qttx[1] = 0;
- rd_stats_tmp.skip = 0;
- }
-
- av1_merge_rd_stats(&rd_stats_qttx, &rd_stats_tmp);
-
- if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) {
- rd_stats_qttx.rate += x->txfm_partition_cost[ctx][0];
- }
- rd_stats_qttx.rate += x->quarter_tx_size_cost[1];
- rd_qttx = RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist);
-#if CONFIG_LV_MAP
- eobs_qttx[0] = p->txb_entropy_ctx[0];
- eobs_qttx[1] = p->txb_entropy_ctx[block_offset_qttx];
-#else
- eobs_qttx[0] = p->eobs[0];
- eobs_qttx[1] = p->eobs[block_offset_qttx];
-#endif
- if (rd_qttx < this_rd) {
- is_qttx_picked = 1;
- this_rd = rd_qttx;
- rd_stats->rate = rd_stats_qttx.rate;
- rd_stats->dist = rd_stats_qttx.dist;
- rd_stats->sse = rd_stats_qttx.sse;
- rd_stats->skip = rd_stats_qttx.skip;
- rd_stats->rdcost = rd_stats_qttx.rdcost;
- }
- av1_get_entropy_contexts(plane_bsize, 0, pd, ta, tl);
- }
-#endif
}
if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH && tx_split_prune_flag == 0
@@ -4267,39 +4085,15 @@
if (this_rd < sum_rd) {
int idx, idy;
-#if CONFIG_RECT_TX_EXT
- TX_SIZE tx_size_selected = is_qttx_picked ? quarter_txsize : tx_size;
-#else
TX_SIZE tx_size_selected = tx_size;
-#endif
-#if CONFIG_RECT_TX_EXT
- if (is_qttx_picked) {
- assert(blk_row == 0 && blk_col == 0 && plane == 0);
#if CONFIG_LV_MAP
- p->txb_entropy_ctx[0] = eobs_qttx[0];
- p->txb_entropy_ctx[block_offset_qttx] = eobs_qttx[1];
-#else
- p->eobs[0] = eobs_qttx[0];
- p->eobs[block_offset_qttx] = eobs_qttx[1];
-#endif
- } else {
-#endif
-#if CONFIG_LV_MAP
- p->txb_entropy_ctx[block] = tmp_eob;
+ p->txb_entropy_ctx[block] = tmp_eob;
#else
p->eobs[block] = tmp_eob;
#endif
-#if CONFIG_RECT_TX_EXT
- }
-#endif
av1_set_txb_context(x, plane, block, tx_size_selected, pta, ptl);
-#if CONFIG_RECT_TX_EXT
- if (is_qttx_picked)
- av1_set_txb_context(x, plane, block_offset_qttx, tx_size_selected,
- pta + blk_col_offset, ptl + blk_row_offset);
-#endif // CONFIG_RECT_TX_EXT
txfm_partition_update(tx_above + blk_col, tx_left + blk_row, tx_size,
tx_size);
@@ -4312,16 +4106,7 @@
mbmi->txk_type[txk_idx] = best_tx_type;
#endif
if (this_rd == INT64_MAX) *is_cost_valid = 0;
-#if CONFIG_RECT_TX_EXT
- if (is_qttx_picked) {
- x->blk_skip[plane][0] = skip_qttx[0];
- x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = skip_qttx[1];
- } else {
-#endif
- x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip;
-#if CONFIG_RECT_TX_EXT
- }
-#endif
+ x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip;
} else {
*rd_stats = sum_rd_stats;
if (sum_rd == INT64_MAX) *is_cost_valid = 0;
