commit 243b66bc32032503319de9177a6a02265eb6d25c
author Jingning Han <jingning@google.com> Fri Jun 23 12:11:47 2017 -0700
committer Jingning Han <jingning@google.com> Wed Jun 28 04:07:09 2017 +0000
tree c973ab8c76c192757b5e3f3fa0f2178b8185696e
parent 814737209499bf13ac1c4c0cf7cbc1e6b9bad477

Support rectangular tx_type coding in var-tx

Support the transform block kernel coding for rectangular
transform block size in var-tx. This integrates txk-sel with
var-tx.

Change-Id: I9a8edd84812168f56c79b78cc9af34f6304b1d54

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 7a629fe..638548f 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1095,6 +1095,7 @@
                                   int block, TX_SIZE tx_size) {
   const MODE_INFO *const mi = xd->mi[0];
   const MB_MODE_INFO *const mbmi = &mi->mbmi;
+
 #if CONFIG_INTRABC && (!CONFIG_EXT_TX || CONFIG_TXK_SEL)
   // TODO(aconverse@google.com): Handle INTRABC + EXT_TX + TXK_SEL
   if (is_intrabc_block(mbmi)) return DCT_DCT;
@@ -1158,7 +1159,7 @@
   (void)tx_size;
   TX_TYPE tx_type;
   if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
-      mbmi->tx_size >= TX_32X32) {
+      txsize_sqr_map[tx_size] >= TX_32X32) {
     tx_type = DCT_DCT;
   } else {
     tx_type = mbmi->txk_type[block];