Resolve conflict between var-tx and super-tx
This commit aligns the rate-distortion metric for the recursive
transform block partitioning and the super transform. It resolves
the conflicts between these two experiments. The coding performance
gains of the combined experiments (var-tx + super-tx) has been
improved:
derf 0.89% -> 1.9%
hevcmr 1.06% -> 1.8%
stdhd 0.29% -> 1.4%
hevchr 0.80% -> 2.3%
Change-Id: I7e33994ad70c1b2751435620815f867d82172f41
diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 18a4980..9e20000 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -3164,7 +3164,7 @@
#if CONFIG_VAR_TX
xd->above_txfm_context = cm->above_txfm_context + mi_col;
xd->left_txfm_context =
- xd->left_txfm_context_buffer + (mi_row & 0x07);
+ xd->left_txfm_context_buffer + (mi_row & MI_MASK);
restore_context(x, mi_row, mi_col, a, l, sa, sl, ta, tl, bsize);
#else
restore_context(x, mi_row, mi_col, a, l, sa, sl, bsize);
@@ -5188,6 +5188,11 @@
ext_tx_set = get_ext_tx_set(tx_size, bsize, 1);
#endif // CONFIG_EXT_TX
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
+#if CONFIG_VAR_TX
+ ENTROPY_CONTEXT ctxa[16], ctxl[16];
+ const struct macroblockd_plane *const pd = &xd->plane[0];
+ int coeff_ctx = 1;
+#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX
if (!ext_tx_used_inter[ext_tx_set][tx_type])
continue;
@@ -5201,12 +5206,23 @@
continue;
#endif // CONFIG_EXT_TX
mbmi->tx_type = tx_type;
- vp10_txfm_rd_in_plane_supertx(x,
+
#if CONFIG_VAR_TX
- cpi,
-#endif
- &this_rate, &this_dist, &pnskip,
+ this_rate = 0;
+ this_dist = 0;
+ pnsse = 0;
+ pnskip = 1;
+
+ vp10_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl);
+ coeff_ctx = combine_entropy_contexts(ctxa[0], ctxl[0]);
+ vp10_tx_block_rd_b(cpi, x, tx_size,
+ 0, 0, 0, 0,
+ bsize, coeff_ctx,
+ &this_rate, &this_dist, &pnsse, &pnskip);
+#else
+ vp10_txfm_rd_in_plane_supertx(x, &this_rate, &this_dist, &pnskip,
&pnsse, INT64_MAX, 0, bsize, tx_size, 0);
+#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX
if (get_ext_tx_types(tx_size, bsize, 1) > 1 &&
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index f227272..9f21c5d 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -2317,10 +2317,10 @@
}
#if CONFIG_VAR_TX
-static void tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
- int blk_row, int blk_col, int plane, int block,
- int plane_bsize, int coeff_ctx,
- int *rate, int64_t *dist, int64_t *bsse, int *skip) {
+void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
+ int blk_row, int blk_col, int plane, int block,
+ int plane_bsize, int coeff_ctx,
+ int *rate, int64_t *dist, int64_t *bsse, int *skip) {
MACROBLOCKD *xd = &x->e_mbd;
const struct macroblock_plane *const p = &x->plane[plane];
struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -2531,8 +2531,8 @@
if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) {
mbmi->inter_tx_size[tx_idx] = tx_size;
- tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
- plane_bsize, coeff_ctx, rate, dist, bsse, skip);
+ vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
+ plane_bsize, coeff_ctx, rate, dist, bsse, skip);
if ((RDCOST(x->rdmult, x->rddiv, *rate, *dist) >=
RDCOST(x->rdmult, x->rddiv, zero_blk_rate, *bsse) || *skip == 1) &&
@@ -2863,8 +2863,8 @@
break;
}
coeff_ctx = combine_entropy_contexts(ta[0], tl[0]);
- tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
- plane_bsize, coeff_ctx, rate, dist, bsse, skip);
+ vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
+ plane_bsize, coeff_ctx, rate, dist, bsse, skip);
for (i = 0; i < (1 << tx_size); ++i) {
ta[i] = !(p->eobs[block] == 0);
tl[i] = !(p->eobs[block] == 0);
diff --git a/vp10/encoder/rdopt.h b/vp10/encoder/rdopt.h
index 62b0aea..a6394fa 100644
--- a/vp10/encoder/rdopt.h
+++ b/vp10/encoder/rdopt.h
@@ -74,6 +74,13 @@
int64_t best_rd_so_far);
#if CONFIG_SUPERTX
+#if CONFIG_VAR_TX
+void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
+ int blk_row, int blk_col, int plane, int block,
+ int plane_bsize, int coeff_ctx,
+ int *rate, int64_t *dist, int64_t *bsse, int *skip);
+#endif
+
void vp10_txfm_rd_in_plane_supertx(MACROBLOCK *x,
#if CONFIG_VAR_TX
const VP10_COMP *cpi,
