commit 0c70a80feb824acfa22691f4f3f3b040152c4c9d
author Jingning Han <jingning@google.com> Tue Jan 10 15:29:45 2017 -0800
committer Jingning Han <jingning@google.com> Thu Jan 12 00:39:56 2017 +0000
tree bd1f106922c97d18c1f728238fb94c374cb918f2
parent cefb409475de858a8cacc1595a69936647ac37f5

Make txfm block partition context support rectangular blocks

Make the transform block partition context model support the
rectangular transform block size partition. The coding gains
from cb4x4 and var-tx are:
          cb4x4 + var-tx
lowres         4.3%
midres         2.6%

Change-Id: I6cc1413fbf6d7707ca7fd24300623a3f0118be7c

av1/common/onyxc_int.h

diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index f7659c4..bb17c09 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -815,6 +815,17 @@
   TX_SIZE max_tx_size = max_txsize_lookup[bsize];
   int category = TXFM_PARTITION_CONTEXTS - 1;
 
+  // dummy return, not used by others.
+  if (tx_size <= TX_4X4) return 0;
+
+  switch (AOMMAX(block_size_wide[bsize], block_size_high[bsize])) {
+    case 64:
+    case 32: max_tx_size = TX_32X32; break;
+    case 16: max_tx_size = TX_16X16; break;
+    case 8: max_tx_size = TX_8X8; break;
+    default: assert(0);
+  }
+
   if (max_tx_size >= TX_8X8) {
     category = (tx_size != max_tx_size && max_tx_size > TX_8X8) +
                (TX_SIZES - 1 - max_tx_size) * 2;