Refactor the ext-tx experiment
Use common structure for inter and intra tx type information when
possible.
Change-Id: I1fd3bc86033871ffbcc2b496a31dca00b7d64b31
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 88a31bb..0dcf735 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -121,16 +121,24 @@
void av1_encode_token_init(void) {
#if CONFIG_EXT_TX
int s;
-#endif // CONFIG_EXT_TX
-#if CONFIG_EXT_TX
for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
- av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
+ av1_tokens_from_tree(ext_tx_inter_encodings[s],
+ av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]]);
}
for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
- av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
+ av1_tokens_from_tree(ext_tx_intra_encodings[s],
+ av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]]);
+ }
+ for (s = 1; s < EXT_TX_SET_TYPES; ++s) {
+ av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s],
+ av1_ext_tx_tree[s]);
}
#else
av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
+ /* This hack is necessary because the four TX_TYPES are not consecutive,
+ e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
+ structure. */
+ av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
#endif // CONFIG_EXT_TX
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
@@ -158,19 +166,6 @@
an in-order traversal of the av1_switchable_interp_tree structure. */
av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
av1_switchable_interp_tree);
-/* This hack is necessary because the four TX_TYPES are not consecutive,
- e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
- structure. */
-#if CONFIG_EXT_TX
- for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
- av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
- av1_ext_tx_intra_tree[s]);
- for (s = 1; s < EXT_TX_SETS_INTER; ++s)
- av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
- av1_ext_tx_inter_tree[s]);
-#else
- av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
-#endif
}
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
@@ -1607,23 +1602,23 @@
if (tx_type == MRC_DCT)
assert(mbmi->valid_mrc_mask && "Invalid MRC mask");
#endif // CONFIG_MRC_TX
-
+ const TxSetType tx_set_type = get_ext_tx_set_type(
+ tx_size, bsize, is_inter, cm->reduced_tx_set_used);
const int eset =
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
// eset == 0 should correspond to a set with only DCT_DCT and there
// is no need to send the tx_type
assert(eset > 0);
+ assert(av1_ext_tx_used[tx_set_type][tx_type]);
if (is_inter) {
- assert(ext_tx_used_inter[eset][tx_type]);
- aom_write_symbol(w, av1_ext_tx_inter_ind[eset][tx_type],
+ aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][tx_type],
ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
- ext_tx_cnt_inter[eset]);
+ av1_num_ext_tx_set[tx_set_type]);
} else if (ALLOW_INTRA_EXT_TX) {
- assert(ext_tx_used_intra[eset][tx_type]);
aom_write_symbol(
- w, av1_ext_tx_intra_ind[eset][tx_type],
+ w, av1_ext_tx_ind[tx_set_type][tx_type],
ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
- ext_tx_cnt_intra[eset]);
+ av1_num_ext_tx_set[tx_set_type]);
}
}
#else
@@ -3082,10 +3077,12 @@
!skip) {
const int eset =
get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
+ const int tx_set_type =
+ get_ext_tx_set_type(supertx_size, bsize, 1, cm->reduced_tx_set_used);
if (eset > 0) {
- aom_write_symbol(w, av1_ext_tx_inter_ind[eset][mbmi->tx_type],
+ aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][mbmi->tx_type],
ec_ctx->inter_ext_tx_cdf[eset][supertx_size],
- ext_tx_cnt_inter[eset]);
+ av1_num_ext_tx_set[tx_set_type]);
}
}
#else
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 201f858..40f0064 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6073,16 +6073,20 @@
const int eset =
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (eset > 0) {
+ const TxSetType tx_set_type = get_ext_tx_set_type(
+ tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (is_inter) {
update_cdf(fc->inter_ext_tx_cdf[eset][txsize_sqr_map[tx_size]],
- av1_ext_tx_inter_ind[eset][tx_type], ext_tx_cnt_inter[eset]);
+ av1_ext_tx_ind[tx_set_type][tx_type],
+ av1_num_ext_tx_set[tx_set_type]);
++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type];
} else {
++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode]
[tx_type];
update_cdf(
fc->intra_ext_tx_cdf[eset][txsize_sqr_map[tx_size]][mbmi->mode],
- av1_ext_tx_intra_ind[eset][tx_type], ext_tx_cnt_intra[eset]);
+ av1_ext_tx_ind[tx_set_type][tx_type],
+ av1_num_ext_tx_set[tx_set_type]);
}
}
}
@@ -7327,9 +7331,6 @@
TX_SIZE tx_size;
MB_MODE_INFO *mbmi;
TX_TYPE tx_type, best_tx_nostx;
-#if CONFIG_EXT_TX
- int ext_tx_set;
-#endif // CONFIG_EXT_TX
int tmp_rate_tx = 0, skip_tx = 0;
int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX;
@@ -7399,7 +7400,9 @@
tx_size = max_txsize_lookup[bsize];
av1_subtract_plane(x, bsize, 0);
#if CONFIG_EXT_TX
- ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used);
+ int ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used);
+ const TxSetType tx_set_type =
+ get_ext_tx_set_type(tx_size, bsize, 1, cm->reduced_tx_set_used);
#endif // CONFIG_EXT_TX
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
#if CONFIG_VAR_TX
@@ -7410,7 +7413,7 @@
#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX
- if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
+ if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
#else
if (tx_size >= TX_32X32 && tx_type != DCT_DCT) continue;
#endif // CONFIG_EXT_TX
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 1300578..b06c43d 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2044,12 +2044,11 @@
}
#if CONFIG_EXT_TX
- int is_inter = is_inter_block(mbmi);
- int ext_tx_set = get_ext_tx_set(get_min_tx_size(tx_size), mbmi->sb_type,
- is_inter, cm->reduced_tx_set_used);
- if (!(is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) &&
- !(!is_inter && ext_tx_used_intra[ext_tx_set][tx_type]))
- continue;
+ const int is_inter = is_inter_block(mbmi);
+ const TxSetType tx_set_type =
+ get_ext_tx_set_type(get_min_tx_size(tx_size), mbmi->sb_type, is_inter,
+ cm->reduced_tx_set_used);
+ if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
#endif // CONFIG_EXT_TX
RD_STATS this_rd_stats;
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 8a6ce0b..fd8b96e 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -179,15 +179,16 @@
if (use_inter_ext_tx_for_txsize[s][i]) {
av1_cost_tokens_from_cdf(x->inter_tx_type_costs[s][i],
fc->inter_ext_tx_cdf[s][i],
- av1_ext_tx_inter_inv[s]);
+ av1_ext_tx_inv[av1_ext_tx_set_type_inter[s]]);
}
}
for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
if (use_intra_ext_tx_for_txsize[s][i]) {
- for (j = 0; j < INTRA_MODES; ++j)
- av1_cost_tokens_from_cdf(x->intra_tx_type_costs[s][i][j],
- fc->intra_ext_tx_cdf[s][i][j],
- av1_ext_tx_intra_inv[s]);
+ for (j = 0; j < INTRA_MODES; ++j) {
+ av1_cost_tokens_from_cdf(
+ x->intra_tx_type_costs[s][i][j], fc->intra_ext_tx_cdf[s][i][j],
+ av1_ext_tx_inv[av1_ext_tx_set_type_intra[s]]);
+ }
}
}
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 019e1f5..cabfd7c 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2311,7 +2311,7 @@
}
}
-// #TODO(angiebird): use this function whenever it's possible
+// TODO(angiebird): use this function whenever it's possible
int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane,
TX_SIZE tx_size, TX_TYPE tx_type) {
@@ -2442,10 +2442,10 @@
if (max_tx_size >= TX_32X32 && tx_size == TX_4X4) return 1;
#if CONFIG_EXT_TX
const AV1_COMMON *const cm = &cpi->common;
- int ext_tx_set =
- get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used);
+ const TxSetType tx_set_type =
+ get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used);
+ if (!av1_ext_tx_used[tx_set_type][tx_type]) return 1;
if (is_inter) {
- if (!ext_tx_used_inter[ext_tx_set][tx_type]) return 1;
if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
if (!do_tx_type_search(tx_type, prune)) return 1;
}
@@ -2453,7 +2453,6 @@
if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) return 1;
}
- if (!ext_tx_used_intra[ext_tx_set][tx_type]) return 1;
}
#else // CONFIG_EXT_TX
if (tx_size >= TX_32X32 && tx_type != DCT_DCT) return 1;
@@ -2494,9 +2493,6 @@
const int is_inter = is_inter_block(mbmi);
int prune = 0;
const int plane = 0;
-#if CONFIG_EXT_TX
- int ext_tx_set;
-#endif // CONFIG_EXT_TX
av1_invalid_rd_stats(rd_stats);
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
@@ -2504,8 +2500,10 @@
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX
- ext_tx_set =
+ int ext_tx_set =
get_ext_tx_set(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used);
+ const TxSetType tx_set_type =
+ get_ext_tx_set_type(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used);
#endif // CONFIG_EXT_TX
if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE)
@@ -2526,12 +2524,12 @@
#endif // CONFIG_PVQ
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
+ if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
RD_STATS this_rd_stats;
if (is_inter) {
if (x->use_default_inter_tx_type &&
tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size))
continue;
- if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
if (!do_tx_type_search(tx_type, prune)) continue;
}
@@ -2542,7 +2540,6 @@
if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue;
}
- if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue;
}
mbmi->tx_type = tx_type;
@@ -2696,10 +2693,9 @@
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
RD_STATS this_rd_stats;
- int ext_tx_set =
- get_ext_tx_set(rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
- if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) ||
- (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) {
+ const TxSetType tx_set_type = get_ext_tx_set_type(
+ rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
+ if (av1_ext_tx_used[tx_set_type][tx_type]) {
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type,
rect_tx_size);
ref_best_rd = AOMMIN(rd, ref_best_rd);
@@ -2745,10 +2741,9 @@
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
const TX_SIZE tx_size = quarter_txsize_lookup[bs];
RD_STATS this_rd_stats;
- int ext_tx_set =
- get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used);
- if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) ||
- (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) {
+ const TxSetType tx_set_type =
+ get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used);
+ if (av1_ext_tx_used[tx_set_type][tx_type]) {
rd =
txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, tx_size);
if (rd < best_rd) {
@@ -5271,6 +5266,8 @@
RD_STATS rd_stats_stack[4];
#endif // CONFIG_EXT_PARTITION
#if CONFIG_EXT_TX
+ const TxSetType tx_set_type = get_ext_tx_set_type(
+ max_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
const int ext_tx_set =
get_ext_tx_set(max_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
#endif // CONFIG_EXT_TX
@@ -5315,8 +5312,8 @@
continue;
#endif // CONFIG_MRC_TX
#if CONFIG_EXT_TX
+ if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
if (is_inter) {
- if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
if (!do_tx_type_search(tx_type, prune)) continue;
}
@@ -5324,7 +5321,6 @@
if (!ALLOW_INTRA_EXT_TX && bsize >= BLOCK_8X8) {
if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue;
}
- if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue;
}
#else // CONFIG_EXT_TX
if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE &&
