Define block_signals_txsize function This returns true if a block signals tx_size in the stream and uses it in the bitstream writing code and the decoder. Note that we can't quite use it in pack_inter_mode_mvs when CONFIG_VAR_TX && !CONFIG_RECT_TX but I've switched the code to using it the rest of the time since rect-tx is adopted and eventually the other code path should be deleted. Also use the helper function in tx_size_cost in rdopt.c, where the test was wrong and caused underestimates of block costs. (Specifically, the code that subtracts tx_size_cost from this_rate_tokenonly in rd_pick_intra_sby_mode ended up subtracting zero for a 4x8 block). The behaviour of the decoder should be unchanged. The only change in the encoder's behaviour should be in tx_size_cost where it should now match the rest of the code. Change-Id: I97236c9ce444993afe01ac5c6f4a0bb9e5049217
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 9ee553f..24c37ba 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -892,6 +892,14 @@ static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; } #endif // CONFIG_RECT_TX +static INLINE int block_signals_txsize(BLOCK_SIZE bsize) { +#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX + return bsize > BLOCK_4X4; +#else + return bsize >= BLOCK_8X8; +#endif +} + #if CONFIG_MRC_TX #define USE_MRC_INTRA 0 #define USE_MRC_INTER 1
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 6005be5..351524c 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -520,12 +520,8 @@ const TX_MODE tx_mode = cm->tx_mode; const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4; -#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX - if (bsize > BLOCK_4X4) -#else - if (bsize >= BLOCK_8X8) -#endif // CONFIG_CB4X4 && CONFIG_VAR_TX - { + + if (block_signals_txsize(bsize)) { if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) { const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] : intra_tx_size_cat_lookup[bsize];
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index c03db84..515d48e 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -348,12 +348,7 @@ const BLOCK_SIZE bsize = mbmi->sb_type; FRAME_CONTEXT *ec_ctx = xd->tile_ctx; (void)cm; -// For sub8x8 blocks the tx_size symbol does not need to be sent -#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX - if (bsize > BLOCK_4X4) { -#else - if (bsize >= BLOCK_8X8) { -#endif + if (block_signals_txsize(bsize)) { const TX_SIZE tx_size = mbmi->tx_size; const int is_inter = is_inter_block(mbmi); const TX_SIZE tx_size_ctx = get_tx_size_context(xd); @@ -1754,14 +1749,10 @@ write_is_inter(cm, xd, mbmi->segment_id, w, is_inter); if (cm->tx_mode == TX_MODE_SELECT && -#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX) -#if CONFIG_RECT_TX - bsize > BLOCK_4X4 && -#else +#if CONFIG_CB4X4 && CONFIG_VAR_TX && !CONFIG_RECT_TX (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) && -#endif // CONFIG_RECT_TX #else - bsize >= BLOCK_8X8 && + block_signals_txsize(bsize) && #endif #if CONFIG_SUPERTX !supertx_enabled && @@ -2157,15 +2148,7 @@ } int enable_tx_size = cm->tx_mode == TX_MODE_SELECT && -#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX) -#if CONFIG_RECT_TX - bsize > BLOCK_4X4 && -#else - bsize >= BLOCK_8X8 && -#endif // CONFIG_RECT_TX -#else - bsize >= BLOCK_8X8 && -#endif + block_signals_txsize(bsize) && !xd->lossless[mbmi->segment_id]; #if CONFIG_INTRABC
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 47badf9..c3553d6 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -2219,16 +2219,7 @@ const MACROBLOCKD *const xd = &x->e_mbd; const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; - const int tx_select = cm->tx_mode == TX_MODE_SELECT && -#if CONFIG_EXT_PARTITION_TYPES - // Currently these block shapes can only use 4x4 - // transforms - mbmi->sb_type != BLOCK_4X16 && - mbmi->sb_type != BLOCK_16X4 && -#endif - mbmi->sb_type >= BLOCK_8X8; - - if (tx_select) { + if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) { const int is_inter = is_inter_block(mbmi); const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] : intra_tx_size_cat_lookup[bsize]; @@ -3060,7 +3051,8 @@ if (tokenonly_rd_stats.rate == INT_MAX) continue; this_rate = tokenonly_rd_stats.rate + palette_mode_cost; this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist); - if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) { + if (!xd->lossless[mbmi->segment_id] && + block_signals_txsize(mbmi->sb_type)) { tokenonly_rd_stats.rate -= tx_size_cost(cpi, x, bsize, mbmi->tx_size); } if (this_rd < *best_rd) { @@ -4212,7 +4204,8 @@ this_rate = this_rate_tokenonly + bmode_costs[mbmi->mode]; - if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) { + if (!xd->lossless[mbmi->segment_id] && + block_signals_txsize(mbmi->sb_type)) { // super_block_yrd above includes the cost of the tx_size in the // tokenonly rate, but for intra blocks, tx_size is always coded // (prediction granularity), so we account for it in the full rate, @@ -10867,7 +10860,7 @@ av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0); } - if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) { + if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(bsize)) { // super_block_yrd above includes the cost of the tx_size in the // tokenonly rate, but for intra blocks, tx_size is always coded // (prediction granularity), so we account for it in the full rate,