commit 3daa4fda6cb0b77515fcbc15f6755a7dac32fc17
author Jingning Han <jingning@google.com> Fri Jan 20 10:33:50 2017 -0800
committer Jingning Han <jingning@google.com> Sat Jan 21 21:04:27 2017 +0000
tree a6a9c1fa14133535b289604d5d2b3e0da7dc9783
parent 9303b941d86a30e653bab2769ecfae7f1cc24f26

Support rectangular tx size in cb4x4 mode

This commit makes ext-tx and rect-tx experiments supported in the
cb4x4 mode. It resolves an enc/dec mismatch issue when all the
transform experiments are enabled.

The coding gains are
        ext-tx + rect-tx   cb4x4    vartx     total
lowres      4.0%           2.3%      0.5%     6.9%

The encoding speed is about the same when cb4x4 and vartx are
further enabled.

BUG=aomedia:139

Change-Id: I3fdabc6d5de23ceb78ac0751a9bf7332ebc0a3ac

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 0d65a15..ca43152 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -427,8 +427,12 @@
                                    aom_writer *w) {
   const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   const BLOCK_SIZE bsize = mbmi->sb_type;
-  // For sub8x8 blocks the tx_size symbol does not need to be sent
+// For sub8x8 blocks the tx_size symbol does not need to be sent
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+  if (bsize > BLOCK_4X4) {
+#else
   if (bsize >= BLOCK_8X8) {
+#endif
     const TX_SIZE tx_size = mbmi->tx_size;
     const int is_inter = is_inter_block(mbmi);
     const int tx_size_ctx = get_tx_size_context(xd);
@@ -1335,8 +1339,8 @@
       aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd));
 
   if (cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_CB4X4 && CONFIG_VAR_TX
-      (bsize >= BLOCK_8X8 || (bsize >= BLOCK_4X4 && is_inter && !skip)) &&
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+      bsize > BLOCK_4X4 &&
 #else
       bsize >= BLOCK_8X8 &&
 #endif
@@ -1700,7 +1704,12 @@
   write_skip(cm, xd, mbmi->segment_id, mi, w);
 #endif
 
-  if (bsize >= BLOCK_8X8 && cm->tx_mode == TX_MODE_SELECT &&
+  if (cm->tx_mode == TX_MODE_SELECT &&
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+      bsize > BLOCK_4X4 &&
+#else
+      bsize >= BLOCK_8X8 &&
+#endif
       !xd->lossless[mbmi->segment_id])
     write_selected_tx_size(cm, xd, w);
 

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index b7d8e84..c6b9f85 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5683,10 +5683,8 @@
     TX_SIZE tx_size = mbmi->tx_size;
 #endif
     if (cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_CB4X4 && CONFIG_VAR_TX
-        (mbmi->sb_type >= BLOCK_8X8 ||
-         (mbmi->sb_type >= BLOCK_4X4 && is_inter &&
-          !(mbmi->skip || seg_skip))) &&
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+        mbmi->sb_type > BLOCK_4X4 &&
 #else
         mbmi->sb_type >= BLOCK_8X8 &&
 #endif
@@ -5783,7 +5781,7 @@
 #if CONFIG_VAR_TX
   if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4
-      mbmi->sb_type >= BLOCK_4X4 &&
+      mbmi->sb_type > BLOCK_4X4 &&
 #else
       mbmi->sb_type >= BLOCK_8X8 &&
 #endif
@@ -5795,7 +5793,7 @@
     if (is_inter)
       tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, is_inter);
     else
-      tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
+      tx_size = (bsize > BLOCK_4X4) ? tx_size : TX_4X4;
     mbmi->tx_size = tx_size;
     set_txfm_ctxs(tx_size, xd->n8_w, xd->n8_h, (mbmi->skip || seg_skip), xd);
   }