commit 3daa4fda6cb0b77515fcbc15f6755a7dac32fc17
author Jingning Han <jingning@google.com> Fri Jan 20 10:33:50 2017 -0800
committer Jingning Han <jingning@google.com> Sat Jan 21 21:04:27 2017 +0000
tree a6a9c1fa14133535b289604d5d2b3e0da7dc9783
parent 9303b941d86a30e653bab2769ecfae7f1cc24f26

Support rectangular tx size in cb4x4 mode

This commit makes ext-tx and rect-tx experiments supported in the
cb4x4 mode. It resolves an enc/dec mismatch issue when all the
transform experiments are enabled.

The coding gains are
        ext-tx + rect-tx   cb4x4    vartx     total
lowres      4.0%           2.3%      0.5%     6.9%

The encoding speed is about the same when cb4x4 and vartx are
further enabled.

BUG=aomedia:139

Change-Id: I3fdabc6d5de23ceb78ac0751a9bf7332ebc0a3ac

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 25c7eac..85fd4ca 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -419,8 +419,8 @@
   const TX_MODE tx_mode = cm->tx_mode;
   const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
   if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
-#if CONFIG_CB4X4 && CONFIG_VAR_TX
-  if ((bsize > BLOCK_4X4 && is_inter) || bsize >= BLOCK_8X8) {
+#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+  if (bsize > BLOCK_4X4) {
 #else
   if (bsize >= BLOCK_8X8) {
 #endif  // CONFIG_CB4X4 && CONFIG_VAR_TX
@@ -1991,8 +1991,7 @@
 
     if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4
-        (bsize >= BLOCK_8X8 ||
-         (bsize >= BLOCK_4X4 && inter_block && !mbmi->skip)) &&
+        bsize > BLOCK_4X4 &&
 #else
         bsize >= BLOCK_8X8 &&
 #endif