commit 3daa4fda6cb0b77515fcbc15f6755a7dac32fc17
author Jingning Han <jingning@google.com> Fri Jan 20 10:33:50 2017 -0800
committer Jingning Han <jingning@google.com> Sat Jan 21 21:04:27 2017 +0000
tree a6a9c1fa14133535b289604d5d2b3e0da7dc9783
parent 9303b941d86a30e653bab2769ecfae7f1cc24f26

Support rectangular tx size in cb4x4 mode

This commit makes ext-tx and rect-tx experiments supported in the
cb4x4 mode. It resolves an enc/dec mismatch issue when all the
transform experiments are enabled.

The coding gains are
        ext-tx + rect-tx   cb4x4    vartx     total
lowres      4.0%           2.3%      0.5%     6.9%

The encoding speed is about the same when cb4x4 and vartx are
further enabled.

BUG=aomedia:139

Change-Id: I3fdabc6d5de23ceb78ac0751a9bf7332ebc0a3ac

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 0d65a15..ca43152 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -427,8 +427,12 @@
                                    aom_writer *w) {
   const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   const BLOCK_SIZE bsize = mbmi->sb_type;
-  // For sub8x8 blocks the tx_size symbol does not need to be sent
+// For sub8x8 blocks the tx_size symbol does not need to be sent
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+  if (bsize > BLOCK_4X4) {
+#else
   if (bsize >= BLOCK_8X8) {
+#endif
     const TX_SIZE tx_size = mbmi->tx_size;
     const int is_inter = is_inter_block(mbmi);
     const int tx_size_ctx = get_tx_size_context(xd);
@@ -1335,8 +1339,8 @@
       aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd));
 
   if (cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_CB4X4 && CONFIG_VAR_TX
-      (bsize >= BLOCK_8X8 || (bsize >= BLOCK_4X4 && is_inter && !skip)) &&
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+      bsize > BLOCK_4X4 &&
 #else
       bsize >= BLOCK_8X8 &&
 #endif
@@ -1700,7 +1704,12 @@
   write_skip(cm, xd, mbmi->segment_id, mi, w);
 #endif
 
-  if (bsize >= BLOCK_8X8 && cm->tx_mode == TX_MODE_SELECT &&
+  if (cm->tx_mode == TX_MODE_SELECT &&
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+      bsize > BLOCK_4X4 &&
+#else
+      bsize >= BLOCK_8X8 &&
+#endif
       !xd->lossless[mbmi->segment_id])
     write_selected_tx_size(cm, xd, w);