Support rectangular tx_size in recursive txfm partition This commit supports the recursive transform block partition to start with both square and rectangular transform block sizes. Change-Id: Idaf29d50cb1f4876b43e5ba82c2609708c6b1926
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 692a570..f1288e6 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -601,7 +601,19 @@ static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode, int is_inter) { const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode]; +#if CONFIG_VAR_TX + const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize]; + + if (!is_inter || bsize < BLOCK_8X8) + return AOMMIN(max_txsize_lookup[bsize], largest_tx_size); + + if (txsize_sqr_map[max_tx_size] <= largest_tx_size) + return max_tx_size; + else + return largest_tx_size; +#else const TX_SIZE max_tx_size = max_txsize_lookup[bsize]; +#endif #if CONFIG_EXT_TX && CONFIG_RECT_TX if (!is_inter) {
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index a2c583a..c2cad0d 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -1481,7 +1481,7 @@ // TODO(jingning): This can be simplified for decoder performance. const BLOCK_SIZE plane_bsize = get_plane_block_size(AOMMAX(bsize, BLOCK_8X8), pd); - const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const int bh_var_tx = tx_size_high_unit[max_tx_size]; const int bw_var_tx = tx_size_wide_unit[max_tx_size]; #if CONFIG_EXT_TX && CONFIG_RECT_TX
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index b7fa43e..b560f02 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -1841,7 +1841,7 @@ xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK); if (bsize >= BLOCK_8X8 && cm->tx_mode == TX_MODE_SELECT && !mbmi->skip && inter_block) { - const TX_SIZE max_tx_size = max_txsize_lookup[bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize]; const int bh = tx_size_high_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size]; const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 2f1c9a1..3d1c3af 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -1280,7 +1280,7 @@ !(is_inter && skip) && !xd->lossless[segment_id]) { #if CONFIG_VAR_TX if (is_inter) { // This implies skip flag is 0. - const TX_SIZE max_tx_size = max_txsize_lookup[bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize]; const int bh = tx_size_high_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size]; const int width = num_4x4_blocks_wide_lookup[bsize]; @@ -1912,7 +1912,7 @@ if (is_inter_block(mbmi)) #endif { - const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; int block = 0; const int step = tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size];
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 95d5cb9..d79fb96 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -5166,7 +5166,7 @@ MACROBLOCKD *xd = &x->e_mbd; const int mi_width = num_4x4_blocks_wide_lookup[plane_bsize]; const int mi_height = num_4x4_blocks_high_lookup[plane_bsize]; - TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const int bh = tx_size_high_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size]; int idx, idy; @@ -5224,7 +5224,7 @@ int mi_row, int mi_col) { const int mi_width = num_4x4_blocks_wide_lookup[plane_bsize]; const int mi_height = num_4x4_blocks_high_lookup[plane_bsize]; - TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const int bh = tx_size_high_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size]; int idx, idy;
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index 8e873fe..d37654a 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -1182,7 +1182,7 @@ const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0]; - const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size]; const int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0]; const int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 0f831db..526cb2e 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -3366,7 +3366,7 @@ const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); const int mi_width = num_4x4_blocks_wide_lookup[plane_bsize]; const int mi_height = num_4x4_blocks_high_lookup[plane_bsize]; - const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const int bh = tx_size_high_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size]; int idx, idy; @@ -3704,7 +3704,7 @@ const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); const int mi_width = num_4x4_blocks_wide_lookup[plane_bsize]; const int mi_height = num_4x4_blocks_high_lookup[plane_bsize]; - const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const int bh = tx_size_high_unit[max_tx_size]; const int bw = tx_size_wide_unit[max_tx_size]; int idx, idy;
diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c index 73f63d3..0012559 100644 --- a/av1/encoder/tokenize.c +++ b/av1/encoder/tokenize.c
@@ -613,8 +613,6 @@ const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane); TX_SIZE plane_tx_size; - assert(tx_size < TX_SIZES); - if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; plane_tx_size = @@ -685,7 +683,7 @@ const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0]; const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0]; - const TX_SIZE max_tx_size = max_txsize_lookup[plane_bsize]; + const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize]; const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size]; int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0]; int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];