Support rectangular tx_size in recursive txfm partition
This commit supports the recursive transform block partition to
start with both square and rectangular transform block sizes.
Change-Id: Idaf29d50cb1f4876b43e5ba82c2609708c6b1926
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 692a570..f1288e6 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -601,7 +601,19 @@
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
int is_inter) {
const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
+#if CONFIG_VAR_TX
+ const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
+
+ if (!is_inter || bsize < BLOCK_8X8)
+ return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
+
+ if (txsize_sqr_map[max_tx_size] <= largest_tx_size)
+ return max_tx_size;
+ else
+ return largest_tx_size;
+#else
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
+#endif
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (!is_inter) {
