Fix for var_tx context update The tx_partition_set_contexts function changes tx_size even for blocks coded with a rectangular transform. This causes an internal rd inconsistency when using all of CONFIG_VAR_TX, CONFIG_RECT_TX, CONFIG_EXT_TX. Change-Id: Ia45d4a8893b0961534219bb96d9652719038c7a1
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 3f93d53..41a71bb 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -5236,11 +5236,14 @@ #if CONFIG_VAR_TX if (cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8 && is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) { - if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col); #if CONFIG_EXT_TX && CONFIG_RECT_TX if (is_rect_tx(mbmi->tx_size)) { set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd); + } else { + if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col); } +#else + if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX } else { TX_SIZE tx_size;