commit f4e097b48689de73963b4be54554366d0207e269
author Jingning Han <jingning@google.com> Fri Jan 20 09:23:58 2017 -0800
committer Jingning Han <jingning@google.com> Fri Jan 20 18:14:48 2017 +0000
tree a658742cb69da873067ccfb16747481fddc36f09
parent 3eb713e287456dc19f496a39a85beab6dfb753c7

Fix rectangular tables in cb4x4 mode

Account for the additional block sizes in these tables.

Change-Id: Iae940f28671714caaf32432940752958ef66f6d5

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index fa82409..7642b63 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -670,6 +670,11 @@
 #if CONFIG_RECT_TX
 static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
   static const char LUT[BLOCK_SIZES] = {
+#if CONFIG_CB4X4
+    0,  // BLOCK_2X2
+    0,  // BLOCK_2X4
+    0,  // BLOCK_4X2
+#endif
     0,  // BLOCK_4X4
     1,  // BLOCK_4X8
     1,  // BLOCK_8X4

av1/common/common_data.h

diff --git a/av1/common/common_data.h b/av1/common/common_data.h
index bcfdf1a..a80ca5e 100644
--- a/av1/common/common_data.h
+++ b/av1/common/common_data.h
@@ -495,11 +495,16 @@
 #if CONFIG_CB4X4
   // 2X2,             2X4,                4X2,
   INT32_MIN,          INT32_MIN,          INT32_MIN,
-#endif
+  //                                      4X4,
+                                          INT32_MIN,
+  // 4X8,             8X4,                8X8,
+  TX_8X8 - TX_8X8,    TX_8X8 - TX_8X8,    TX_8X8 - TX_8X8,
+#else
   //                                      4X4
                                           INT32_MIN,
   // 4X8,             8X4,                8X8
   INT32_MIN,          INT32_MIN,          TX_8X8 - TX_8X8,
+#endif  // CONFIG_CB4X4
   // 8X16,            16X8,               16X16
   TX_16X16 - TX_8X8,  TX_16X16 - TX_8X8,  TX_16X16 - TX_8X8,
   // 16X32,           32X16,              32X32