Fix config flags among var-tx/ext-tx/rect-tx
BUG=aomedia:396
Change-Id: I1b7f6dddb06eac9e341518a95a0d71465e613e9e
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index dc433a2..a924ed8 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -602,7 +602,6 @@
#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
#if CONFIG_RECT_TX
-
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
#endif // CONFIG_RECT_TX
@@ -776,13 +775,13 @@
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
int is_inter) {
const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
-#if CONFIG_VAR_TX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
+#if (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
#else
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
-#endif // CONFIG_VAR_TX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
+#endif // (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
(void)is_inter;
-#if CONFIG_VAR_TX
+#if CONFIG_VAR_TX && CONFIG_RECT_TX
#if CONFIG_CB4X4
if (bsize == BLOCK_4X4)
return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
@@ -802,7 +801,7 @@
}
#else
return AOMMIN(max_tx_size, largest_tx_size);
-#endif // CONFIG_VAR_TX
+#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
}
#if CONFIG_EXT_INTRA
diff --git a/av1/common/common_data.h b/av1/common/common_data.h
index 7730411..6369386 100644
--- a/av1/common/common_data.h
+++ b/av1/common/common_data.h
@@ -455,7 +455,7 @@
#endif // CONFIG_TX64X64
};
-#if CONFIG_RECT_TX
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES] = {
#if CONFIG_CB4X4
// 2X2, 2X4, 4X2,
@@ -489,9 +489,9 @@
};
#else
#define max_txsize_rect_lookup max_txsize_lookup
-#endif // CONFIG_RECT_TX
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
-#if (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular
// block which may use a rectangular transform, in which case it is
// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8
@@ -564,7 +564,7 @@
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
};
-#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c
index ade5222..01a3480 100644
--- a/av1/common/reconintra.c
+++ b/av1/common/reconintra.c
@@ -2262,7 +2262,7 @@
predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride,
dst, dst_stride, col_off, row_off, plane);
} else {
-#if (CONFIG_VAR_TX || CONFIG_RECT_TX) || \
+#if (CONFIG_RECT_TX && (CONFIG_VAR_TX || CONFIG_EXT_TX)) || \
(CONFIG_EXT_INTER && USE_RECT_INTERINTRA)
#if CONFIG_AOM_HIGHBITDEPTH
uint16_t tmp16[MAX_SB_SIZE];
@@ -2384,7 +2384,8 @@
}
#else
assert(0);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
+#endif // (CONFIG_RECT_TX && (CONFIG_VAR_TX || CONFIG_EXT_TX)) ||
+ // (CONFIG_EXT_INTER && USE_RECT_INTERINTRA)
}
}
diff --git a/av1/common/scan.c b/av1/common/scan.c
index 9a09056..2377d90 100644
--- a/av1/common/scan.c
+++ b/av1/common/scan.c
@@ -6500,14 +6500,14 @@
case TX_8X8: return fc->non_zero_prob_8X8[tx_type];
case TX_16X16: return fc->non_zero_prob_16X16[tx_type];
case TX_32X32: return fc->non_zero_prob_32X32[tx_type];
-#if CONFIG_VAR_TX || CONFIG_RECT_TX
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->non_zero_prob_4X8[tx_type];
case TX_8X4: return fc->non_zero_prob_8X4[tx_type];
case TX_8X16: return fc->non_zero_prob_8X16[tx_type];
case TX_16X8: return fc->non_zero_prob_16X8[tx_type];
case TX_16X32: return fc->non_zero_prob_16X32[tx_type];
case TX_32X16: return fc->non_zero_prob_32X16[tx_type];
-#endif
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
@@ -6522,14 +6522,14 @@
case TX_8X8: return fc->scan_8X8[tx_type];
case TX_16X16: return fc->scan_16X16[tx_type];
case TX_32X32: return fc->scan_32X32[tx_type];
-#if CONFIG_VAR_TX || CONFIG_RECT_TX
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->scan_4X8[tx_type];
case TX_8X4: return fc->scan_8X4[tx_type];
case TX_8X16: return fc->scan_8X16[tx_type];
case TX_16X8: return fc->scan_16X8[tx_type];
case TX_16X32: return fc->scan_16X32[tx_type];
case TX_32X16: return fc->scan_32X16[tx_type];
-#endif
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
@@ -6544,14 +6544,14 @@
case TX_8X8: return fc->iscan_8X8[tx_type];
case TX_16X16: return fc->iscan_16X16[tx_type];
case TX_32X32: return fc->iscan_32X32[tx_type];
-#if CONFIG_VAR_TX || CONFIG_RECT_TX
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->iscan_4X8[tx_type];
case TX_8X4: return fc->iscan_8X4[tx_type];
case TX_8X16: return fc->iscan_8X16[tx_type];
case TX_16X8: return fc->iscan_16X8[tx_type];
case TX_16X32: return fc->iscan_16X32[tx_type];
case TX_32X16: return fc->iscan_32X16[tx_type];
-#endif
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
@@ -6566,14 +6566,14 @@
case TX_8X8: return fc->nb_8X8[tx_type];
case TX_16X16: return fc->nb_16X16[tx_type];
case TX_32X32: return fc->nb_32X32[tx_type];
-#if CONFIG_VAR_TX || CONFIG_RECT_TX
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->nb_4X8[tx_type];
case TX_8X4: return fc->nb_8X4[tx_type];
case TX_8X16: return fc->nb_8X16[tx_type];
case TX_16X8: return fc->nb_16X8[tx_type];
case TX_16X32: return fc->nb_16X32[tx_type];
case TX_32X16: return fc->nb_32X16[tx_type];
-#endif
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
@@ -6588,14 +6588,14 @@
case TX_8X8: return counts->non_zero_count_8X8[tx_type];
case TX_16X16: return counts->non_zero_count_16X16[tx_type];
case TX_32X32: return counts->non_zero_count_32X32[tx_type];
-#if CONFIG_VAR_TX || CONFIG_RECT_TX
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return counts->non_zero_count_4x8[tx_type];
case TX_8X4: return counts->non_zero_count_8x4[tx_type];
case TX_8X16: return counts->non_zero_count_8x16[tx_type];
case TX_16X8: return counts->non_zero_count_16x8[tx_type];
case TX_16X32: return counts->non_zero_count_16x32[tx_type];
case TX_32X16: return counts->non_zero_count_32x16[tx_type];
-#endif
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
@@ -6774,11 +6774,11 @@
TX_SIZE tx_size;
TX_TYPE tx_type;
for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) {
-#if !(CONFIG_VAR_TX || CONFIG_RECT_TX)
- if (tx_size >= TX_SIZES) continue;
-#else
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (tx_size > TX_32X16) continue;
-#endif
+#else
+ if (tx_size >= TX_SIZES) continue;
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
uint32_t *non_zero_prob = get_non_zero_prob(cm->fc, tx_size, tx_type);
const int tx2d_size = tx_size_2d[tx_size];
@@ -6798,11 +6798,11 @@
void av1_adapt_scan_order(AV1_COMMON *cm) {
TX_SIZE tx_size;
for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) {
-#if !(CONFIG_VAR_TX || CONFIG_RECT_TX)
- if (tx_size >= TX_SIZES) continue;
-#else
+#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (tx_size > TX_32X16) continue;
-#endif // !(CONFIG_VAR_TX || CONFIG_RECT_TX)
+#else
+ if (tx_size >= TX_SIZES) continue;
+#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
TX_TYPE tx_type;
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
update_scan_prob(cm, tx_size, tx_type, ADAPT_SCAN_UPDATE_RATE_16);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 53717f1..c7da753 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -448,13 +448,9 @@
const TX_MODE tx_mode = cm->tx_mode;
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
-#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
-#if CONFIG_RECT_TX
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
if (bsize > BLOCK_4X4) {
#else
- if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) {
-#endif // CONFIG_RECT_TX
-#else
if (bsize >= BLOCK_8X8) {
#endif // CONFIG_CB4X4 && CONFIG_VAR_TX
if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) {
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 587134f..3c63639 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -456,13 +456,9 @@
FRAME_CONTEXT *ec_ctx = cm->fc;
#endif
// For sub8x8 blocks the tx_size symbol does not need to be sent
-#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
-#if CONFIG_RECT_TX
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
if (bsize > BLOCK_4X4) {
#else
- if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter_block(mbmi))) {
-#endif // CONFIG_RECT_TX
-#else
if (bsize >= BLOCK_8X8) {
#endif
const TX_SIZE tx_size = mbmi->tx_size;
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 82f55bb..5c6a4d2 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5771,14 +5771,9 @@
TX_SIZE tx_size = mbmi->tx_size;
#endif
if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id] &&
-#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
-#if CONFIG_RECT_TX
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
mbmi->sb_type > BLOCK_4X4 &&
#else
- (mbmi->sb_type >= BLOCK_8X8 ||
- (mbmi->sb_type > BLOCK_4X4 && is_inter)) &&
-#endif
-#else
mbmi->sb_type >= BLOCK_8X8 &&
#endif
!(is_inter && (mbmi->skip || seg_skip))) {
