commit a1e48dccf2123b6b8bb37e9c444a2c1e5a006394
author Yue Chen <yuec@google.com> Mon Aug 29 17:29:33 2016 -0700
committer Yue Chen <yuec@google.com> Tue Sep 06 11:26:15 2016 -0700
tree d6480619523e9af705162413562eec90fa665c64
parent 5ebdf40d77f16757f6ec2020665d38a8821266cc

Make RECT_TX(>=8x8) work with VAR_TX

Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.

Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0

av1/common/entropymode.c

diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index b2cb6ff..e1593e3 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -857,6 +857,12 @@
       },
     };
 
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
+// the probability of (0) using recursive square tx partition vs.
+// (1) biggest rect tx for 4X8-8X4/8X16-16X8/16X32-32X16 blocks
+static const aom_prob default_rect_tx_prob[TX_SIZES - 1] = { 192, 192, 192 };
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
+
 int av1_get_palette_color_context(const uint8_t *color_map, int cols, int r,
                                   int c, int n, int *color_order) {
   int i, j, max, max_idx, temp;
@@ -1298,6 +1304,9 @@
   av1_copy(fc->tx_size_probs, default_tx_size_prob);
 #if CONFIG_VAR_TX
   av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  av1_copy(fc->rect_tx_prob, default_rect_tx_prob);
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
   av1_copy(fc->skip_probs, default_skip_probs);
 #if CONFIG_REF_MV
@@ -1451,6 +1460,15 @@
   }
 #endif  // CONFIG_EXT_INTER
 
+#if CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (cm->tx_mode == TX_MODE_SELECT) {
+    for (i = 0; i < TX_SIZES - 1; ++i) {
+      fc->rect_tx_prob[i] =
+          av1_mode_mv_merge_probs(pre_fc->rect_tx_prob[i], counts->rect_tx[i]);
+    }
+  }
+#endif  // CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
+
   for (i = 0; i < BLOCK_SIZE_GROUPS; i++)
     aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->y_mode_prob[i],
                          counts->y_mode[i], fc->y_mode_prob[i]);
@@ -1478,10 +1496,11 @@
   }
 
 #if CONFIG_VAR_TX
-  if (cm->tx_mode == TX_MODE_SELECT)
+  if (cm->tx_mode == TX_MODE_SELECT) {
     for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
       fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs(
           pre_fc->txfm_partition_prob[i], counts->txfm_partition[i]);
+  }
 #endif
 
   for (i = 0; i < SKIP_CONTEXTS; ++i)