Enable 4:1 -> 2:1 -> 1:1 splits for 4:1 transforms
Also, splits the sub_tx_size_map array into inter and intra
in order to enable the new 4:1 transforms for inter and
intra separately.
Includes refactoring such as removing the intra_tx_size_cat_lookup
array since it is unnecessary, and consolidating the
max_txsize_rect_lookup array for convenience.
Change-Id: I112553bab612dafb973611c87f36a43e1ac4be85
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 1c41637..8ccce76 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -944,10 +944,7 @@
}
static INLINE TX_SIZE get_max_rect_tx_size(BLOCK_SIZE bsize, int is_inter) {
- if (is_inter)
- return max_txsize_rect_lookup[bsize];
- else
- return max_txsize_rect_intra_lookup[bsize];
+ return max_txsize_rect_lookup[is_inter][bsize];
}
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
@@ -1102,18 +1099,31 @@
int depth = 0;
while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
depth++;
- tx_size = sub_tx_size_map[tx_size];
+ tx_size = sub_tx_size_map[is_inter][tx_size];
}
return depth;
}
+static INLINE int bsize_to_tx_size_cat(BLOCK_SIZE bsize, int is_inter) {
+ TX_SIZE tx_size = get_max_rect_tx_size(bsize, is_inter);
+ assert(tx_size != TX_4X4);
+ int depth = 0;
+ while (tx_size != TX_4X4) {
+ depth++;
+ tx_size = sub_tx_size_map[is_inter][tx_size];
+ assert(depth < 10);
+ }
+ assert(depth <= MAX_TX_CATS);
+ return depth - 1;
+}
+
static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize,
int is_inter) {
TX_SIZE ctx_size = get_max_rect_tx_size(bsize, is_inter);
int depth = 0;
while (tx_size != ctx_size) {
depth++;
- ctx_size = sub_tx_size_map[ctx_size];
+ ctx_size = sub_tx_size_map[is_inter][ctx_size];
assert(depth <= MAX_TX_DEPTH);
}
return depth;
@@ -1123,7 +1133,7 @@
int is_inter) {
TX_SIZE max_tx_size = get_max_rect_tx_size(bsize, is_inter);
TX_SIZE tx_size = max_tx_size;
- for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[tx_size];
+ for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[is_inter][tx_size];
return tx_size;
}
diff --git a/av1/common/common_data.h b/av1/common/common_data.h
index d9c6764..181f980 100644
--- a/av1/common/common_data.h
+++ b/av1/common/common_data.h
@@ -647,126 +647,128 @@
#endif // CONFIG_EXT_PARTITION
};
-static const TX_SIZE max_txsize_rect_intra_lookup[BLOCK_SIZES_ALL] = {
- // 2X2, 2X4, 4X2,
- TX_4X4, TX_4X4, TX_4X4,
- // 4X4
- TX_4X4,
- // 4X8, 8X4, 8X8
- TX_4X8, TX_8X4, TX_8X8,
- // 8X16, 16X8, 16X16
- TX_8X16, TX_16X8, TX_16X16,
- // 16X32, 32X16, 32X32
- TX_16X32, TX_32X16, TX_32X32,
+static const TX_SIZE max_txsize_rect_lookup[2][BLOCK_SIZES_ALL] = {
+ {
+ // Intra
+ // 2X2, 2X4, 4X2,
+ TX_4X4, TX_4X4, TX_4X4,
+ // 4X4
+ TX_4X4,
+ // 4X8, 8X4, 8X8
+ TX_4X8, TX_8X4, TX_8X8,
+ // 8X16, 16X8, 16X16
+ TX_8X16, TX_16X8, TX_16X16,
+ // 16X32, 32X16, 32X32
+ TX_16X32, TX_32X16, TX_32X32,
#if CONFIG_TX64X64
- // 32X64, 64X32,
- TX_32X64, TX_64X32,
- // 64X64
- TX_64X64,
+ // 32X64, 64X32,
+ TX_32X64, TX_64X32,
+ // 64X64
+ TX_64X64,
#if CONFIG_EXT_PARTITION
- // 64x128, 128x64, 128x128
- TX_64X64, TX_64X64, TX_64X64,
+ // 64x128, 128x64, 128x128
+ TX_64X64, TX_64X64, TX_64X64,
#endif // CONFIG_EXT_PARTITION
#else
- // 32X64, 64X32,
- TX_32X32, TX_32X32,
- // 64X64
- TX_32X32,
+ // 32X64, 64X32,
+ TX_32X32, TX_32X32,
+ // 64X64
+ TX_32X32,
#if CONFIG_EXT_PARTITION
- // 64x128, 128x64, 128x128
- TX_32X32, TX_32X32, TX_32X32,
+ // 64x128, 128x64, 128x128
+ TX_32X32, TX_32X32, TX_32X32,
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT_INTRA
- // 4x16, 16x4,
- TX_4X16, TX_16X4,
- // 8x32, 32x8
- TX_8X32, TX_32X8,
+ // 4x16, 16x4,
+ TX_4X16, TX_16X4,
+ // 8x32, 32x8
+ TX_8X32, TX_32X8,
#if CONFIG_TX64X64
- // 16x64, 64x16
- TX_16X64, TX_64X16,
+ // 16x64, 64x16
+ TX_16X64, TX_64X16,
#else
- // 16x64, 64x16
- TX_16X32, TX_32X16,
+ // 16x64, 64x16
+ TX_16X32, TX_32X16,
#endif // CONFIG_TX64X64
#else
- // 4x16, 16x4,
- TX_4X8, TX_8X4,
- // 8x32, 32x8
- TX_8X16, TX_16X8,
- // 16x64, 64x16
- TX_16X32, TX_32X16,
+ // 4x16, 16x4,
+ TX_4X8, TX_8X4,
+ // 8x32, 32x8
+ TX_8X16, TX_16X8,
+ // 16x64, 64x16
+ TX_16X32, TX_32X16,
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT_INTRA
#if CONFIG_EXT_PARTITION
#if CONFIG_TX64X64
- // 32x128 128x32
- TX_32X64, TX_64X32
+ // 32x128 128x32
+ TX_32X64, TX_64X32
#else
- // 32x128 128x32
- TX_32X32, TX_32X32
+ // 32x128 128x32
+ TX_32X32, TX_32X32
#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_PARTITION
-};
-
-static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES_ALL] = {
- // 2X2, 2X4, 4X2,
- TX_4X4, TX_4X4, TX_4X4,
- // 4X4
- TX_4X4,
- // 4X8, 8X4, 8X8
- TX_4X8, TX_8X4, TX_8X8,
- // 8X16, 16X8, 16X16
- TX_8X16, TX_16X8, TX_16X16,
- // 16X32, 32X16, 32X32
- TX_16X32, TX_32X16, TX_32X32,
+ }, {
+ // Inter
+ // 2X2, 2X4, 4X2,
+ TX_4X4, TX_4X4, TX_4X4,
+ // 4X4
+ TX_4X4,
+ // 4X8, 8X4, 8X8
+ TX_4X8, TX_8X4, TX_8X8,
+ // 8X16, 16X8, 16X16
+ TX_8X16, TX_16X8, TX_16X16,
+ // 16X32, 32X16, 32X32
+ TX_16X32, TX_32X16, TX_32X32,
#if CONFIG_TX64X64
- // 32X64, 64X32,
- TX_32X64, TX_64X32,
- // 64X64
- TX_64X64,
+ // 32X64, 64X32,
+ TX_32X64, TX_64X32,
+ // 64X64
+ TX_64X64,
#if CONFIG_EXT_PARTITION
- // 64x128, 128x64, 128x128
- TX_64X64, TX_64X64, TX_64X64,
+ // 64x128, 128x64, 128x128
+ TX_64X64, TX_64X64, TX_64X64,
#endif // CONFIG_EXT_PARTITION
#else
- // 32X64, 64X32,
- TX_32X32, TX_32X32,
- // 64X64
- TX_32X32,
+ // 32X64, 64X32,
+ TX_32X32, TX_32X32,
+ // 64X64
+ TX_32X32,
#if CONFIG_EXT_PARTITION
- // 64x128, 128x64, 128x128
- TX_32X32, TX_32X32, TX_32X32,
+ // 64x128, 128x64, 128x128
+ TX_32X32, TX_32X32, TX_32X32,
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
- // 4x16, 16x4, 8x32
- TX_4X16, TX_16X4, TX_8X32,
- // 32x8
- TX_32X8,
+ // 4x16, 16x4, 8x32
+ TX_4X16, TX_16X4, TX_8X32,
+ // 32x8
+ TX_32X8,
#if CONFIG_TX64X64
- // 16x64, 64x16
- TX_16X64, TX_64X16,
+ // 16x64, 64x16
+ TX_16X64, TX_64X16,
#else
- // 16x64, 64x16
- TX_16X32, TX_32X16,
+ // 16x64, 64x16
+ TX_16X32, TX_32X16,
#endif // CONFIG_TX64X64
#else
- // 4x16, 16x4, 8x32
- TX_4X8, TX_8X4, TX_8X16,
- // 32x8
- TX_16X8,
- // 16x64, 64x16
- TX_16X32, TX_32X16,
+ // 4x16, 16x4, 8x32
+ TX_4X8, TX_8X4, TX_8X16,
+ // 32x8
+ TX_16X8,
+ // 16x64, 64x16
+ TX_16X32, TX_32X16,
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
#if CONFIG_EXT_PARTITION
#if CONFIG_TX64X64
- // 32x128 128x32
- TX_32X64, TX_64X32
+ // 32x128 128x32
+ TX_32X64, TX_64X32
#else
- // 32x128 128x32
- TX_32X32, TX_32X32
+ // 32x128 128x32
+ TX_32X32, TX_32X32
#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_PARTITION
+ },
};
static const TX_TYPE_1D vtx_tab[TX_TYPES] = {
@@ -781,88 +783,89 @@
IDTX_1D, DCT_1D, IDTX_1D, ADST_1D, IDTX_1D, FLIPADST_1D,
};
-// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular
-// block which may use a rectangular transform, in which case it is
-// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8
#define TXSIZE_CAT_INVALID (-1)
-static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES_ALL] = {
- // 2X2, 2X4, 4X2,
- TXSIZE_CAT_INVALID, TXSIZE_CAT_INVALID, TXSIZE_CAT_INVALID,
- // 4X4,
- TXSIZE_CAT_INVALID,
- // 4X8, 8X4, 8X8,
- TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_8X8 - TX_8X8,
- // 8X16, 16X8, 16X16
- TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_16X16 - TX_8X8,
- // 16X32, 32X16, 32X32
- TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
-#if CONFIG_TX64X64
- // 32X64, 64X32,
- TX_64X64 - TX_8X8, TX_64X64 - TX_8X8,
- // 64X64
- TX_64X64 - TX_8X8,
-#if CONFIG_EXT_PARTITION
- // 64x128, 128x64, 128x128
- TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, TX_64X64 - TX_8X8,
-#endif // CONFIG_EXT_PARTITION
-#else
- // 32X64, 64X32,
- TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
- // 64X64
- TX_32X32 - TX_8X8,
-#if CONFIG_EXT_PARTITION
- // 64x128, 128x64, 128x128
- TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
-#endif // CONFIG_EXT_PARTITION
-#endif // CONFIG_TX64X64
- // TODO(david.barker): Change these if we support rectangular transforms
- // for 4:1 shaped partitions
- // 4x16, 16x4, 8x32
- TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_16X16 - TX_8X8,
- // 32x8, 16x64, 64x16
- TX_16X16 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
-#if CONFIG_EXT_PARTITION
-#if CONFIG_TX64X64
- // 32x128, 128x32
- TX_64X64 - TX_8X8, TX_64X64 - TX_8X8
-#else
- // 32x128, 128x32
- TX_32X32 - TX_8X8, TX_32X32 - TX_8X8
-#endif // CONFIG_TX64X64
-#endif // CONFIG_EXT_PARTITION
-};
-
-#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup
-
/* clang-format on */
-static const TX_SIZE sub_tx_size_map[TX_SIZES_ALL] = {
- TX_4X4, // TX_4X4
- TX_4X4, // TX_8X8
- TX_8X8, // TX_16X16
- TX_16X16, // TX_32X32
+static const TX_SIZE sub_tx_size_map[2][TX_SIZES_ALL] = {
+ {
+ // Intra
+ TX_4X4, // TX_4X4
+ TX_4X4, // TX_8X8
+ TX_8X8, // TX_16X16
+ TX_16X16, // TX_32X32
#if CONFIG_TX64X64
- TX_32X32, // TX_64X64
-#endif // CONFIG_TX64X64
- TX_4X4, // TX_4X8
- TX_4X4, // TX_8X4
- TX_8X8, // TX_8X16
- TX_8X8, // TX_16X8
- TX_16X16, // TX_16X32
- TX_16X16, // TX_32X16
+ TX_32X32, // TX_64X64
+#endif // CONFIG_TX64X64
+ TX_4X4, // TX_4X8
+ TX_4X4, // TX_8X4
+ TX_8X8, // TX_8X16
+ TX_8X8, // TX_16X8
+ TX_16X16, // TX_16X32
+ TX_16X16, // TX_32X16
#if CONFIG_TX64X64
- TX_32X32, // TX_32X64
- TX_32X32, // TX_64X32
-#endif // CONFIG_TX64X64
- TX_4X4, // TX_4X16
- TX_4X4, // TX_16X4
- TX_8X8, // TX_8X32
- TX_8X8, // TX_32X8
+ TX_32X32, // TX_32X64
+ TX_32X32, // TX_64X32
+#endif // CONFIG_TX64X64
+#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT_INTRA
+ TX_4X8, // TX_4X16
+ TX_8X4, // TX_16X4
+ TX_8X16, // TX_8X32
+ TX_16X8, // TX_32X8
#if CONFIG_TX64X64
- TX_16X16, // TX_16X64
- TX_16X16, // TX_64X16
-#endif // CONFIG_TX64X64
+ TX_16X32, // TX_16X64
+ TX_32X16, // TX_64X16
+#endif // CONFIG_TX64X64
+#else
+ TX_4X4, // TX_4X16
+ TX_4X4, // TX_16X4
+ TX_8X8, // TX_8X32
+ TX_8X8, // TX_32X8
+#if CONFIG_TX64X64
+ TX_16X16, // TX_16X64
+ TX_16X16, // TX_64X16
+#endif // CONFIG_TX64X64
+#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT_INTRA
+ },
+ {
+ // Inter
+ TX_4X4, // TX_4X4
+ TX_4X4, // TX_8X8
+ TX_8X8, // TX_16X16
+ TX_16X16, // TX_32X32
+#if CONFIG_TX64X64
+ TX_32X32, // TX_64X64
+#endif // CONFIG_TX64X64
+ TX_4X4, // TX_4X8
+ TX_4X4, // TX_8X4
+ TX_8X8, // TX_8X16
+ TX_8X8, // TX_16X8
+ TX_16X16, // TX_16X32
+ TX_16X16, // TX_32X16
+#if CONFIG_TX64X64
+ TX_32X32, // TX_32X64
+ TX_32X32, // TX_64X32
+#endif // CONFIG_TX64X64
+#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
+ TX_4X8, // TX_4X16
+ TX_8X4, // TX_16X4
+ TX_8X16, // TX_8X32
+ TX_16X8, // TX_32X8
+#if CONFIG_TX64X64
+ TX_16X32, // TX_16X64
+ TX_32X16, // TX_64X16
+#endif // CONFIG_TX64X64
+#else
+ TX_4X4, // TX_4X16
+ TX_4X4, // TX_16X4
+ TX_8X8, // TX_8X32
+ TX_8X8, // TX_32X8
+#if CONFIG_TX64X64
+ TX_16X16, // TX_16X64
+ TX_16X16, // TX_64X16
+#endif // CONFIG_TX64X64
+#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
+ },
};
static const TX_SIZE txsize_horz_map[TX_SIZES_ALL] = {
@@ -1733,7 +1736,7 @@
#if CONFIG_TX64X64
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
#endif // CONFIG_TX64X64
- { { TX_4X8, TX_4X8 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X8, TX_4X4 }, { TX_4X8, TX_4X4 } },
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
{ { TX_4X8, TX_4X8 }, { TX_4X4, TX_4X4 } },
{ { TX_4X8, TX_4X8 }, { TX_4X4, TX_4X4 } },
@@ -1743,7 +1746,7 @@
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
#endif // CONFIG_TX64X64
- { { TX_4X16, TX_4X8 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X16, TX_4X8 }, { TX_4X16, TX_4X8 } },
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
{ { TX_4X16, TX_4X8 }, { TX_4X4, TX_4X4 } },
{ { TX_4X8, TX_4X8 }, { TX_4X4, TX_4X4 } },
@@ -1762,7 +1765,7 @@
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
#endif // CONFIG_TX64X64
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
- { { TX_8X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
+ { { TX_8X4, TX_8X4 }, { TX_4X4, TX_4X4 } },
{ { TX_8X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
{ { TX_8X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
{ { TX_8X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
@@ -1772,7 +1775,7 @@
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
#endif // CONFIG_TX64X64
{ { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
- { { TX_16X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
+ { { TX_16X4, TX_16X4 }, { TX_8X4, TX_8X4 } },
{ { TX_8X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
{ { TX_16X4, TX_4X4 }, { TX_8X4, TX_4X4 } },
#if CONFIG_TX64X64
@@ -1791,7 +1794,7 @@
#endif // CONFIG_TX64X64
{ { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
{ { TX_8X4, TX_8X4 }, { TX_4X4, TX_4X4 } },
- { { TX_8X16, TX_8X16 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X16, TX_8X8 }, { TX_4X16, TX_4X8 } },
{ { TX_8X8, TX_8X8 }, { TX_4X8, TX_4X8 } },
{ { TX_8X16, TX_8X16 }, { TX_4X8, TX_4X8 } },
{ { TX_8X16, TX_8X16 }, { TX_4X8, TX_4X8 } },
@@ -1820,7 +1823,7 @@
{ { TX_4X8, TX_4X4 }, { TX_4X8, TX_4X4 } },
{ { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
{ { TX_8X8, TX_8X4 }, { TX_8X8, TX_8X4 } },
- { { TX_16X8, TX_8X4 }, { TX_16X8, TX_8X4 } },
+ { { TX_16X8, TX_16X4 }, { TX_8X8, TX_8X4 } },
{ { TX_16X8, TX_8X4 }, { TX_16X8, TX_8X4 } },
{ { TX_16X8, TX_8X4 }, { TX_16X8, TX_8X4 } },
#if CONFIG_TX64X64
@@ -1849,7 +1852,7 @@
{ { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
{ { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
{ { TX_16X8, TX_16X8 }, { TX_8X8, TX_8X8 } },
- { { TX_16X32, TX_16X32 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X32, TX_16X16 }, { TX_8X32, TX_8X16 } },
{ { TX_16X16, TX_16X16 }, { TX_8X16, TX_8X16 } },
#if CONFIG_TX64X64
{ { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } },
@@ -1878,7 +1881,7 @@
{ { TX_8X16, TX_8X8 }, { TX_8X16, TX_8X8 } },
{ { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
{ { TX_16X16, TX_16X8 }, { TX_16X16, TX_16X8 } },
- { { TX_32X16, TX_16X8 }, { TX_32X16, TX_16X8 } },
+ { { TX_32X16, TX_32X8 }, { TX_16X16, TX_16X8 } },
#if CONFIG_TX64X64
{ { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } },
{ { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } },
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 785b99c..6e4e2fe 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -285,7 +285,7 @@
pd->dst.stride, max_scan_line, eob, cm->reduced_tx_set_used);
*eob_total += eob;
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
const int bsw = tx_size_wide_unit[sub_txs];
@@ -532,7 +532,7 @@
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
- if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
+ if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const int bh_var_tx = tx_size_high_unit[max_tx_size];
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 4d7da60..ebaf3bc 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -448,7 +448,7 @@
is_split = aom_read_symbol(r, ec_ctx->txfm_partition_cdf[ctx], 2, ACCT_STR);
if (is_split) {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
@@ -495,8 +495,7 @@
// TODO(debargha): Clean up the logic here. This function should only
// be called for intra.
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
- const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
- : intra_tx_size_cat_lookup[bsize];
+ const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, is_inter);
const int max_depths = bsize_to_max_depth(bsize, 0);
const int ctx = get_tx_size_context(xd);
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
@@ -957,7 +956,7 @@
const TX_SIZE mtx_size =
get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, inter_block);
const TX_SIZE tx_size =
- inter_block ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
+ inter_block ? AOMMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
: mbmi->tx_size;
#endif // !CONFIG_TXK_SEL
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 1b75dc9..bfdf4be 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -230,7 +230,7 @@
xd->left_txfm_context + blk_row, tx_size, tx_size);
// TODO(yuec): set correct txfm partition update for qttx
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
@@ -262,9 +262,9 @@
if (block_signals_txsize(bsize)) {
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
- const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const int depth = tx_size_to_depth(tx_size, bsize, 0);
const int max_depths = bsize_to_max_depth(bsize, 0);
+ const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, 0);
assert(depth >= 0 && depth <= max_depths);
assert(!is_inter_block(mbmi));
@@ -536,7 +536,7 @@
token_stats->cost += tmp_token_stats.cost;
#endif
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
@@ -592,7 +592,7 @@
token_stats->cost += tmp_token_stats.cost;
#endif
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
@@ -1111,7 +1111,7 @@
const TX_SIZE mtx_size =
get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, is_inter);
const TX_SIZE tx_size =
- is_inter ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
+ is_inter ? AOMMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
: mbmi->tx_size;
#endif // !CONFIG_TXK_SEL
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
@@ -1963,7 +1963,7 @@
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
- if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
+ if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const int step =
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index e930bf4..df01ed0 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4424,7 +4424,7 @@
allow_update_cdf) {
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
- const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
+ const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, 0);
const int depth = tx_size_to_depth(tx_size, bsize, 0);
const int max_depths = bsize_to_max_depth(bsize, 0);
update_cdf(fc->tx_size_cdf[tx_size_cat][tx_size_ctx], depth,
@@ -4590,7 +4590,7 @@
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
@@ -4659,7 +4659,7 @@
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int i;
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 8e7888f6..c98bfef 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -685,7 +685,7 @@
encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
} else {
assert(tx_size < TX_SIZES_ALL);
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
// This is the square transform block partition entry point.
@@ -807,7 +807,7 @@
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
- if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
+ if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 379b284..dae2186 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2322,8 +2322,7 @@
if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) {
const int is_inter = is_inter_block(mbmi);
- const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
- : intra_tx_size_cat_lookup[bsize];
+ const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, is_inter);
const int depth = tx_size_to_depth(tx_size, bsize, is_inter);
const int tx_size_ctx = get_tx_size_context(xd);
int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
@@ -2617,7 +2616,8 @@
}
last_rd = INT64_MAX;
- for (n = start_tx; depth <= MAX_TX_DEPTH; depth++, n = sub_tx_size_map[n]) {
+ for (n = start_tx; depth <= MAX_TX_DEPTH;
+ depth++, n = sub_tx_size_map[0][n]) {
TX_TYPE tx_start = DCT_DCT;
TX_TYPE tx_end = TX_TYPES;
#if CONFIG_TXK_SEL
@@ -3938,7 +3938,7 @@
}
if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH && tx_split_prune_flag == 0) {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
int sub_step = bsw * bsh;
@@ -4360,7 +4360,7 @@
txfm_partition_update(tx_above + blk_col, tx_left + blk_row, tx_size,
tx_size);
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int step = bsh * bsw;
@@ -4587,7 +4587,7 @@
int cur_tx_depth = 0;
uint8_t parent_idx_buf[MAX_SB_SQUARE] = { 0 };
- int cur_tx_size = max_txsize_rect_lookup[bsize];
+ int cur_tx_size = max_txsize_rect_lookup[1][bsize];
while (cur_tx_depth <= MAX_VARTX_DEPTH) {
const BLOCK_SIZE cur_tx_bsize = txsize_to_bsize[cur_tx_size];
const int cur_tx_bw = block_size_wide[cur_tx_bsize];
@@ -4641,7 +4641,7 @@
++cur_rd_info_idx;
}
}
- cur_tx_size = sub_tx_size_map[cur_tx_size];
+ cur_tx_size = sub_tx_size_map[1][cur_tx_size];
++cur_tx_depth;
}
return 1;
@@ -4823,7 +4823,7 @@
TX_TYPE tx_type, best_tx_type = DCT_DCT;
const int is_inter = is_inter_block(mbmi);
TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
- TX_SIZE best_tx = max_txsize_rect_lookup[bsize];
+ TX_SIZE best_tx = max_txsize_rect_lookup[1][bsize];
TX_SIZE best_min_tx_size = TX_SIZES_ALL;
uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
TX_TYPE txk_start = DCT_DCT;
@@ -4836,7 +4836,7 @@
int idx, idy;
int prune = 0;
// Get the tx_size 1 level down
- TX_SIZE min_tx_size = sub_tx_size_map[max_txsize_rect_lookup[bsize]];
+ TX_SIZE min_tx_size = sub_tx_size_map[1][max_txsize_rect_lookup[1][bsize]];
const TxSetType tx_set_type = get_ext_tx_set_type(
min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
int within_border = mi_row >= xd->tile.mi_row_start &&
@@ -5001,7 +5001,7 @@
plane_bsize, ta, tl, rd_stats, fast, NULL);
av1_set_txb_context(x, plane, block, tx_size, ta, tl);
} else {
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
const int bsw = tx_size_wide_unit[sub_txs];
@@ -5062,7 +5062,7 @@
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
- if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
+ if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const int bh = tx_size_high_unit[max_tx_size];
@@ -10619,7 +10619,7 @@
// Set up tx_size related variables for skip-specific loop filtering.
best_mbmode.tx_size = block_signals_txsize(bsize)
? tx_size_from_tx_mode(bsize, cm->tx_mode, 1)
- : max_txsize_rect_lookup[bsize];
+ : max_txsize_rect_lookup[1][bsize];
{
const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
const int height = block_size_high[bsize] >> tx_size_high_log2[0];
diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c
index 7d2f6e5..38e78f3 100644
--- a/av1/encoder/tokenize.c
+++ b/av1/encoder/tokenize.c
@@ -559,7 +559,7 @@
#endif
} else {
// Half the block size in transform block unit.
- const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+ const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int step = bsw * bsh;
@@ -640,7 +640,7 @@
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
- if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
+ if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
