commit 243b66bc32032503319de9177a6a02265eb6d25c
author Jingning Han <jingning@google.com> Fri Jun 23 12:11:47 2017 -0700
committer Jingning Han <jingning@google.com> Wed Jun 28 04:07:09 2017 +0000
tree c973ab8c76c192757b5e3f3fa0f2178b8185696e
parent 814737209499bf13ac1c4c0cf7cbc1e6b9bad477

Support rectangular tx_type coding in var-tx

Support the transform block kernel coding for rectangular
transform block size in var-tx. This integrates txk-sel with
var-tx.

Change-Id: I9a8edd84812168f56c79b78cc9af34f6304b1d54

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index d05936a..f896745 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2059,6 +2059,10 @@
                      TX_TYPE tx_type) {
   if (plane > 0) return 0;
 
+#if CONFIG_VAR_TX
+  tx_size = get_min_tx_size(tx_size);
+#endif
+
   const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   const int is_inter = is_inter_block(mbmi);
 #if CONFIG_EXT_TX
@@ -4152,6 +4156,13 @@
   MACROBLOCKD *xd = &x->e_mbd;
   const struct macroblock_plane *const p = &x->plane[plane];
   struct macroblockd_plane *const pd = &xd->plane[plane];
+
+#if CONFIG_TXK_SEL
+  av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
+                      tx_size, a, l, 0, rd_stats);
+  return;
+#endif
+
   int64_t tmp;
   tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
   PLANE_TYPE plane_type = get_plane_type(plane);
@@ -4626,10 +4637,12 @@
                                     [mbmi->tx_type];
     }
   }
-#else   // CONFIG_EXT_TX
+#else  // CONFIG_EXT_TX
+#if !CONFIG_TXK_SEL
   if (mbmi->min_tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id])
     rd_stats->rate +=
         cpi->inter_tx_type_costs[mbmi->min_tx_size][mbmi->tx_type];
+#endif
 #endif  // CONFIG_EXT_TX
 
   if (rd_stats->skip)