Define block_signals_txsize function
This returns true if a block signals tx_size in the stream and uses it
in the bitstream writing code and the decoder.
Note that we can't quite use it in pack_inter_mode_mvs when
CONFIG_VAR_TX && !CONFIG_RECT_TX but I've switched the code to using
it the rest of the time since rect-tx is adopted and eventually the
other code path should be deleted.
Also use the helper function in tx_size_cost in rdopt.c, where the
test was wrong and caused underestimates of block
costs. (Specifically, the code that subtracts tx_size_cost from
this_rate_tokenonly in rd_pick_intra_sby_mode ended up subtracting
zero for a 4x8 block).
The behaviour of the decoder should be unchanged. The only change in
the encoder's behaviour should be in tx_size_cost where it should now
match the rest of the code.
Change-Id: I97236c9ce444993afe01ac5c6f4a0bb9e5049217
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index c03db84..515d48e 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -348,12 +348,7 @@
const BLOCK_SIZE bsize = mbmi->sb_type;
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
(void)cm;
-// For sub8x8 blocks the tx_size symbol does not need to be sent
-#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
- if (bsize > BLOCK_4X4) {
-#else
- if (bsize >= BLOCK_8X8) {
-#endif
+ if (block_signals_txsize(bsize)) {
const TX_SIZE tx_size = mbmi->tx_size;
const int is_inter = is_inter_block(mbmi);
const TX_SIZE tx_size_ctx = get_tx_size_context(xd);
@@ -1754,14 +1749,10 @@
write_is_inter(cm, xd, mbmi->segment_id, w, is_inter);
if (cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
-#if CONFIG_RECT_TX
- bsize > BLOCK_4X4 &&
-#else
+#if CONFIG_CB4X4 && CONFIG_VAR_TX && !CONFIG_RECT_TX
(bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) &&
-#endif // CONFIG_RECT_TX
#else
- bsize >= BLOCK_8X8 &&
+ block_signals_txsize(bsize) &&
#endif
#if CONFIG_SUPERTX
!supertx_enabled &&
@@ -2157,15 +2148,7 @@
}
int enable_tx_size = cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
-#if CONFIG_RECT_TX
- bsize > BLOCK_4X4 &&
-#else
- bsize >= BLOCK_8X8 &&
-#endif // CONFIG_RECT_TX
-#else
- bsize >= BLOCK_8X8 &&
-#endif
+ block_signals_txsize(bsize) &&
!xd->lossless[mbmi->segment_id];
#if CONFIG_INTRABC
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 47badf9..c3553d6 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2219,16 +2219,7 @@
const MACROBLOCKD *const xd = &x->e_mbd;
const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
- const int tx_select = cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_EXT_PARTITION_TYPES
- // Currently these block shapes can only use 4x4
- // transforms
- mbmi->sb_type != BLOCK_4X16 &&
- mbmi->sb_type != BLOCK_16X4 &&
-#endif
- mbmi->sb_type >= BLOCK_8X8;
-
- if (tx_select) {
+ if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) {
const int is_inter = is_inter_block(mbmi);
const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
@@ -3060,7 +3051,8 @@
if (tokenonly_rd_stats.rate == INT_MAX) continue;
this_rate = tokenonly_rd_stats.rate + palette_mode_cost;
this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
- if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) {
+ if (!xd->lossless[mbmi->segment_id] &&
+ block_signals_txsize(mbmi->sb_type)) {
tokenonly_rd_stats.rate -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
}
if (this_rd < *best_rd) {
@@ -4212,7 +4204,8 @@
this_rate = this_rate_tokenonly + bmode_costs[mbmi->mode];
- if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) {
+ if (!xd->lossless[mbmi->segment_id] &&
+ block_signals_txsize(mbmi->sb_type)) {
// super_block_yrd above includes the cost of the tx_size in the
// tokenonly rate, but for intra blocks, tx_size is always coded
// (prediction granularity), so we account for it in the full rate,
@@ -10867,7 +10860,7 @@
av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0);
}
- if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) {
+ if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(bsize)) {
// super_block_yrd above includes the cost of the tx_size in the
// tokenonly rate, but for intra blocks, tx_size is always coded
// (prediction granularity), so we account for it in the full rate,
