commit 3daa4fda6cb0b77515fcbc15f6755a7dac32fc17
author Jingning Han <jingning@google.com> Fri Jan 20 10:33:50 2017 -0800
committer Jingning Han <jingning@google.com> Sat Jan 21 21:04:27 2017 +0000
tree a6a9c1fa14133535b289604d5d2b3e0da7dc9783
parent 9303b941d86a30e653bab2769ecfae7f1cc24f26

Support rectangular tx size in cb4x4 mode

This commit makes ext-tx and rect-tx experiments supported in the
cb4x4 mode. It resolves an enc/dec mismatch issue when all the
transform experiments are enabled.

The coding gains are
        ext-tx + rect-tx   cb4x4    vartx     total
lowres      4.0%           2.3%      0.5%     6.9%

The encoding speed is about the same when cb4x4 and vartx are
further enabled.

BUG=aomedia:139

Change-Id: I3fdabc6d5de23ceb78ac0751a9bf7332ebc0a3ac

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index b7d8e84..c6b9f85 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5683,10 +5683,8 @@
     TX_SIZE tx_size = mbmi->tx_size;
 #endif
     if (cm->tx_mode == TX_MODE_SELECT &&
-#if CONFIG_CB4X4 && CONFIG_VAR_TX
-        (mbmi->sb_type >= BLOCK_8X8 ||
-         (mbmi->sb_type >= BLOCK_4X4 && is_inter &&
-          !(mbmi->skip || seg_skip))) &&
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+        mbmi->sb_type > BLOCK_4X4 &&
 #else
         mbmi->sb_type >= BLOCK_8X8 &&
 #endif
@@ -5783,7 +5781,7 @@
 #if CONFIG_VAR_TX
   if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4
-      mbmi->sb_type >= BLOCK_4X4 &&
+      mbmi->sb_type > BLOCK_4X4 &&
 #else
       mbmi->sb_type >= BLOCK_8X8 &&
 #endif
@@ -5795,7 +5793,7 @@
     if (is_inter)
       tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, is_inter);
     else
-      tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
+      tx_size = (bsize > BLOCK_4X4) ? tx_size : TX_4X4;
     mbmi->tx_size = tx_size;
     set_txfm_ctxs(tx_size, xd->n8_w, xd->n8_h, (mbmi->skip || seg_skip), xd);
   }