Remove tx_size probability tables.
tx_size is transmitted using CDFs now, so these tables are obsolete.
Also, remove a related tx_size_totals table.
Change-Id: I949b46c1c9499bc556e68c77adbdb5aa3288c9a2
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 1040f1a..87d2863 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -2527,55 +2527,6 @@
1] = { -1, -1, 0, -1, -1,
4, 3, 2, 1 };
-// The transform size is coded as an offset to the smallest transform
-// block size.
-const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)] = {
- {
- // Max tx_size is 8X8
- -0, -1,
- },
- {
- // Max tx_size is 16X16
- -0, 2, -1, -2,
- },
- {
- // Max tx_size is 32X32
- -0, 2, -1, 4, -2, -3,
- },
-#if CONFIG_TX64X64
- {
- // Max tx_size is 64X64
- -0, 2, -1, 4, -2, 6, -3, -4,
- },
-#endif // CONFIG_TX64X64
-};
-
-static const aom_prob default_tx_size_prob[MAX_TX_DEPTH][TX_SIZE_CONTEXTS]
- [MAX_TX_DEPTH] = {
- {
- // Max tx_size is 8X8
- { 100 },
- { 66 },
- },
- {
- // Max tx_size is 16X16
- { 20, 152 },
- { 15, 101 },
- },
- {
- // Max tx_size is 32X32
- { 3, 136, 37 },
- { 5, 52, 13 },
- },
-#if CONFIG_TX64X64
- {
- // Max tx_size is 64X64
- { 1, 64, 136, 127 },
- { 1, 32, 52, 67 },
- },
-#endif // CONFIG_TX64X64
- };
-
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static const aom_prob default_quarter_tx_size_prob = 192;
#endif
@@ -5012,7 +4963,6 @@
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
av1_copy(fc->comp_inter_mode_prob, default_comp_inter_mode_p);
#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
- av1_copy(fc->tx_size_probs, default_tx_size_prob);
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
fc->quarter_tx_size_prob = default_quarter_tx_size_prob;
#endif
@@ -5268,17 +5218,12 @@
}
void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
- int i, j;
+ int i;
FRAME_CONTEXT *fc = cm->fc;
const FRAME_CONTEXT *pre_fc = cm->pre_fc;
const FRAME_COUNTS *counts = &cm->counts;
if (cm->tx_mode == TX_MODE_SELECT) {
- for (i = 0; i < MAX_TX_DEPTH; ++i) {
- for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
- aom_tree_merge_probs(av1_tx_size_tree[i], pre_fc->tx_size_probs[i][j],
- counts->tx_size[i][j], fc->tx_size_probs[i][j]);
- }
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
fc->quarter_tx_size_prob = av1_mode_mv_merge_probs(
pre_fc->quarter_tx_size_prob, counts->quarter_tx_size);
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 02dc5b4..e293b35 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -311,7 +311,6 @@
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS];
#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
- aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH];
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
aom_prob quarter_tx_size_prob;
#endif
@@ -492,10 +491,8 @@
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
unsigned int comp_inter_mode[COMP_INTER_MODE_CONTEXTS][2];
#endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
- // TODO(any): tx_size_totals is only used by the encoder to decide whether
- // to use forward updates for the coeff probs, and as such it does not really
- // belong into this structure.
- unsigned int tx_size_totals[TX_SIZES];
+ // TODO(urvang): Only needed for !CONFIG_VAR_TX case. So can be removed when
+ // CONFIG_VAR_TX flag is removed.
unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
unsigned int quarter_tx_size[2];
@@ -576,7 +573,6 @@
#endif
extern const aom_tree_index
av1_palette_color_index_tree[PALETTE_SIZES][TREE_SIZE(PALETTE_COLORS)];
-extern const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)];
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
extern const aom_tree_index av1_intra_filter_tree[TREE_SIZE(INTRA_FILTERS)];
#endif // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 43d9372..d903d71 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6452,9 +6452,6 @@
#endif
}
- ++td->counts->tx_size_totals[txsize_sqr_map[tx_size]];
- ++td->counts->tx_size_totals[txsize_sqr_map[av1_get_uv_tx_size(
- mbmi, &xd->plane[1])]];
#if !CONFIG_TXK_SEL
av1_update_tx_type_count(cm, xd, bsize, tx_size, td->counts);
#endif
diff --git a/tools/aom_entropy_optimizer.c b/tools/aom_entropy_optimizer.c
index 3107477..cc31189 100644
--- a/tools/aom_entropy_optimizer.c
+++ b/tools/aom_entropy_optimizer.c
@@ -678,7 +678,6 @@
#endif
/* Transform size */
-// TODO(yuec): av1_tx_size_tree has variable sizes, so needs special handling
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
cts_each_dim[0] = 2;
optimize_entropy_table(&fc.quarter_tx_size[0], probsfile, 1, cts_each_dim,
