var_tx+intrabc: Fix infinite recursion
Change-Id: Ibbf0c14934d0bff316cfdf5c252c64b3e68c73e7
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index bdf3854..e377970 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -753,6 +753,7 @@
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
+ assert(sub_txs < tx_size);
int i;
assert(bsl > 0);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index d8cc525..35ebc6c 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1121,6 +1121,16 @@
xd->corrupted |=
!assign_dv(cm, xd, &mbmi->mv[0], &dv_ref, mi_row, mi_col, bsize, r);
+#if CONFIG_VAR_TX
+ // TODO(aconverse@google.com): Evaluate allowing VAR TX on intrabc blocks
+ const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
+ const int height = block_size_high[bsize] >> tx_size_high_log2[0];
+ int idx, idy;
+ for (idy = 0; idy < height; ++idy)
+ for (idx = 0; idx < width; ++idx)
+ mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX && !CONFIG_TXK_SEL
av1_read_tx_type(cm, xd,
#if CONFIG_SUPERTX
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index af9d5d2..15012a9 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -1188,12 +1188,13 @@
if (tx_size == plane_tx_size) {
encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
} else {
+ assert(tx_size < TX_SIZES_ALL);
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ assert(sub_txs < tx_size);
// This is the square transform block partition entry point.
int bsl = tx_size_wide_unit[sub_txs];
int i;
assert(bsl > 0);
- assert(tx_size < TX_SIZES_ALL);
for (i = 0; i < 4; ++i) {
const int offsetr = blk_row + ((i >> 1) * bsl);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 17c52a8..9c164eb 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9417,6 +9417,17 @@
mbmi->rd_stats = rd_stats;
#endif
+#if CONFIG_VAR_TX
+ // TODO(aconverse@google.com): Evaluate allowing VAR TX on intrabc blocks
+ const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
+ const int height = block_size_high[bsize] >> tx_size_high_log2[0];
+ int idx, idy;
+ for (idy = 0; idy < height; ++idy)
+ for (idx = 0; idx < width; ++idx)
+ mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
+ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif // CONFIG_VAR_TX
+
const aom_prob skip_prob = av1_get_skip_prob(cm, xd);
RD_STATS rdc_noskip;