Enable rectangular transforms for UV
Uses an array to map block sizes, y tx sizes, and subsampling
factors to various transform sizes for UV.
Results improve by 0.1-0.2%
Change-Id: Icb58fd96bc7c01a72cbf1332fe2be4d55a0feedc
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 8fcaa60..f035bb5 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -684,25 +684,18 @@
void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
-static INLINE TX_SIZE get_uv_tx_size_impl(TX_SIZE y_tx_size, BLOCK_SIZE bsize,
- int xss, int yss) {
- if (bsize < BLOCK_8X8) {
- return TX_4X4;
- } else {
- const BLOCK_SIZE plane_bsize = ss_size_lookup[bsize][xss][yss];
- return AOMMIN(txsize_sqr_map[y_tx_size], max_txsize_lookup[plane_bsize]);
- }
-}
-
static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,
const struct macroblockd_plane *pd) {
+ TX_SIZE uv_txsize;
#if CONFIG_SUPERTX
if (supertx_enabled(mbmi))
return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
[pd->subsampling_x][pd->subsampling_y];
#endif // CONFIG_SUPERTX
- return get_uv_tx_size_impl(mbmi->tx_size, mbmi->sb_type, pd->subsampling_x,
- pd->subsampling_y);
+ uv_txsize = uv_txsize_lookup[mbmi->sb_type][mbmi->tx_size][pd->subsampling_x]
+ [pd->subsampling_y];
+ assert(uv_txsize != TX_INVALID);
+ return uv_txsize;
}
static INLINE BLOCK_SIZE
diff --git a/av1/common/common_data.h b/av1/common/common_data.h
index f86e1f7..450d962 100644
--- a/av1/common/common_data.h
+++ b/av1/common/common_data.h
@@ -518,6 +518,270 @@
#endif // CONFIG_EXT_PARTITION
};
+#define USE_UV_RECT_TX 1
+static const TX_SIZE uv_txsize_lookup[BLOCK_SIZES][TX_SIZES_ALL][2][2] = {
+ // ss_x == 0 ss_x == 0 ss_x == 1 ss_x == 1
+ // ss_y == 0 ss_y == 1 ss_y == 0 ss_y == 1
+ {
+ // BLOCK_4X4
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+#if CONFIG_EXT_TX
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_4X8
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+#if CONFIG_EXT_TX
+#if CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_4X8, TX_4X4 }, { TX_4X4, TX_4X4 } }, // used
+#else
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, // used
+#endif // CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_8X4
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+#if CONFIG_EXT_TX
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#if CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_8X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, // used
+#else
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } }, // used
+#endif // CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_8X8
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X4 }, { TX_4X8, TX_4X4 } },
+ { { TX_8X4, TX_8X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } },
+ { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } },
+ { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } },
+ { { TX_8X8, TX_8X4 }, { TX_4X8, TX_4X4 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_8X16
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_4X4, TX_4X4 } },
+#if CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_8X16, TX_8X8 }, { TX_4X8, TX_4X8 } }, // used
+#else
+ { { TX_8X8, TX_8X8 }, { TX_4X4, TX_4X4 } }, // used
+#endif // CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_16X8
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_4X4 }, { TX_8X8, TX_4X4 } },
+ { { TX_8X8, TX_4X4 }, { TX_8X8, TX_8X8 } },
+ { { TX_8X8, TX_4X4 }, { TX_8X8, TX_8X8 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X4 }, { TX_4X8, TX_4X4 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#if CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_16X8, TX_8X4 }, { TX_8X8, TX_8X4 } }, // used
+#else
+ { { TX_8X8, TX_4X4 }, { TX_8X8, TX_4X4 } }, // used
+#endif // CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_16X16
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_8X8 }, { TX_8X8, TX_8X8 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X8 }, { TX_8X16, TX_8X8 } },
+ { { TX_16X8, TX_16X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X8 }, { TX_8X16, TX_8X8 } },
+ { { TX_16X16, TX_16X8 }, { TX_8X16, TX_8X8 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_16X32
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_8X8, TX_8X8 } },
+#if CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_16X32, TX_16X16 }, { TX_8X16, TX_8X16 } }, // used
+#else
+ { { TX_16X16, TX_16X16 }, { TX_8X8, TX_8X8 } }, // used
+#endif // CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_32X16
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } },
+ { { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X8 }, { TX_8X16, TX_8X8 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_INVALID, TX_INVALID }, { TX_INVALID, TX_INVALID } },
+#if CONFIG_RECT_TX && USE_UV_RECT_TX
+ { { TX_32X16, TX_16X8 }, { TX_16X16, TX_16X8 } }, // used
+#else
+ { { TX_16X16, TX_8X8 }, { TX_16X16, TX_8X8 } }, // used
+#endif // CONFIG_RECT_TX && USE_UV_RECT_TX
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_32X32
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_16X16 }, { TX_16X16, TX_16X16 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_16X32, TX_16X16 }, { TX_16X32, TX_16X16 } },
+ { { TX_32X16, TX_32X16 }, { TX_16X16, TX_16X16 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_32X64
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_32X32 }, { TX_16X16, TX_16X16 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_16X32, TX_16X32 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X16, TX_32X16 }, { TX_16X16, TX_16X16 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_64X32
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_16X16 }, { TX_32X32, TX_16X16 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_16X32, TX_16X16 }, { TX_16X32, TX_16X16 } },
+ { { TX_32X16, TX_16X16 }, { TX_32X16, TX_16X16 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_64X64
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
+#if CONFIG_EXT_PARTITION
+ },
+ {
+ // BLOCK_64X128
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } },
+ { { TX_32X16, TX_32X16 }, { TX_32X16, TX_32X16 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_128X64
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } },
+ { { TX_32X16, TX_32X16 }, { TX_32X16, TX_32X16 } },
+#endif // CONFIG_EXT_TX
+ },
+ {
+ // BLOCK_128X128
+ { { TX_4X4, TX_4X4 }, { TX_4X4, TX_4X4 } },
+ { { TX_8X8, TX_8X8 }, { TX_8X8, TX_8X8 } },
+ { { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
+ { { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
+#if CONFIG_EXT_TX
+ { { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
+ { { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
+ { { TX_8X16, TX_8X16 }, { TX_8X16, TX_8X16 } },
+ { { TX_16X8, TX_16X8 }, { TX_16X8, TX_16X8 } },
+ { { TX_16X32, TX_16X32 }, { TX_16X32, TX_16X32 } },
+ { { TX_32X16, TX_32X16 }, { TX_32X16, TX_32X16 } },
+#endif // CONFIG_EXT_TX
+#endif // CONFIG_EXT_PARTITION
+ },
+};
+
// Generates 4 bit field in which each bit set to 1 represents
// a blocksize partition 1111 means we split 64x64, 32x32, 16x16
// and 8x8. 1000 means we just split the 64x64 to 32x32
diff --git a/av1/common/enums.h b/av1/common/enums.h
index 9f1bbaa..b1ac2a0 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -137,6 +137,7 @@
#define TX_16X16 ((TX_SIZE)2) // 16x16 transform
#define TX_32X32 ((TX_SIZE)3) // 32x32 transform
#define TX_SIZES ((TX_SIZE)4)
+#define TX_INVALID ((TX_SIZE)255) // Invalid transform size
#if CONFIG_EXT_TX
#define TX_4X8 ((TX_SIZE)4) // 4x8 transform
diff --git a/av1/common/loopfilter.c b/av1/common/loopfilter.c
index 726271e..7086be8 100644
--- a/av1/common/loopfilter.c
+++ b/av1/common/loopfilter.c
@@ -713,7 +713,7 @@
// rectangular transfroms are used with the EXT_TX expt.
const TX_SIZE tx_size_y = txsize_sqr_up_map[mbmi->tx_size];
const TX_SIZE tx_size_uv =
- get_uv_tx_size_impl(mbmi->tx_size, block_size, 1, 1);
+ txsize_sqr_up_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
const int filter_level = get_filter_level(lfi_n, mbmi);
uint64_t *const left_y = &lfm->left_y[tx_size_y];
uint64_t *const above_y = &lfm->above_y[tx_size_y];
@@ -1238,9 +1238,9 @@
// Filter level can vary per MI
if (!(lfl[r][c >> ss_x] = get_filter_level(&cm->lf_info, mbmi))) continue;
- if (tx_size == TX_32X32)
+ if (txsize_sqr_up_map[tx_size] == TX_32X32)
tx_size_mask = 3;
- else if (tx_size == TX_16X16)
+ else if (txsize_sqr_up_map[tx_size] == TX_16X16)
tx_size_mask = 1;
else
tx_size_mask = 0;
@@ -1249,8 +1249,8 @@
if (is_inter_block(mbmi) && !mbmi->skip)
tx_size =
(plane->plane_type == PLANE_TYPE_UV)
- ? get_uv_tx_size_impl(mbmi->inter_tx_size[blk_row][blk_col],
- sb_type, ss_x, ss_y)
+ ? uv_txsize_lookup[sb_type][mbmi->inter_tx_size
+ [blk_row][blk_col]][ss_x][ss_y]
: mbmi->inter_tx_size[blk_row][blk_col];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
