commit 74d0ad844e2a7888020f54794e2dc8a4d6568500
author Peter de Rivaz <peter.derivaz@argondesign.com> Wed Oct 19 11:43:11 2016 +0100
committer Peter de Rivaz <peter.derivaz@argondesign.com> Wed Oct 19 11:43:11 2016 +0100
tree dbac0ce6cf40ad8f08759e26e196e26011b3c9de
parent caf2023ae12bb510cb880eb0ff2944bcf84c7527

Fix for var_tx context update

The tx_partition_set_contexts function changes tx_size even
for blocks coded with a rectangular transform.
This causes an internal rd inconsistency when using all of
CONFIG_VAR_TX, CONFIG_RECT_TX, CONFIG_EXT_TX.

Change-Id: Ia45d4a8893b0961534219bb96d9652719038c7a1

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 3f93d53..41a71bb 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5236,11 +5236,14 @@
 #if CONFIG_VAR_TX
   if (cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8 &&
       is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) {
-    if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
     if (is_rect_tx(mbmi->tx_size)) {
       set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+    } else {
+      if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
     }
+#else
+    if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
   } else {
     TX_SIZE tx_size;