commit a1e48dccf2123b6b8bb37e9c444a2c1e5a006394
author Yue Chen <yuec@google.com> Mon Aug 29 17:29:33 2016 -0700
committer Yue Chen <yuec@google.com> Tue Sep 06 11:26:15 2016 -0700
tree d6480619523e9af705162413562eec90fa665c64
parent 5ebdf40d77f16757f6ec2020665d38a8821266cc

Make RECT_TX(>=8x8) work with VAR_TX

Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.

Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 3bd5adc..7c33aea 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1121,9 +1121,25 @@
       const int width = num_4x4_blocks_wide_lookup[bsize];
       const int height = num_4x4_blocks_high_lookup[bsize];
       int idx, idy;
-      for (idy = 0; idy < height; idy += bs)
-        for (idx = 0; idx < width; idx += bs)
-          write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      if (is_rect_tx_allowed(mbmi)) {
+        int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+
+        aom_write(w, is_rect_tx(mbmi->tx_size),
+                  cm->fc->rect_tx_prob[tx_size_cat]);
+      }
+
+      if (is_rect_tx(mbmi->tx_size)) {
+        set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+      } else {
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+        for (idy = 0; idy < height; idy += bs)
+          for (idx = 0; idx < width; idx += bs)
+            write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     } else {
       set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
       write_selected_tx_size(cm, xd, w);
@@ -1650,7 +1666,7 @@
       TX_SIZE tx_size =
           plane ? get_uv_tx_size(mbmi, &xd->plane[plane]) : mbmi->tx_size;
 
-      if (is_inter_block(mbmi) && tx_size < TX_SIZES) {
+      if (is_inter_block(mbmi) && !is_rect_tx(tx_size)) {
 #else
       if (is_inter_block(mbmi)) {
 #endif
@@ -3271,6 +3287,13 @@
 
 #if CONFIG_VAR_TX
   update_txfm_partition_probs(cm, header_bc, counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (cm->tx_mode == TX_MODE_SELECT) {
+    for (i = 1; i < TX_SIZES - 1; ++i)
+      av1_cond_prob_diff_update(header_bc, &fc->rect_tx_prob[i],
+                                counts->rect_tx[i]);
+  }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
 
   update_skip_probs(cm, header_bc, counts);