commit e57d632205c663b052e99f24433f1b9c018215a1
author Jingning Han <jingning@google.com> Mon Jul 03 18:50:25 2017 -0700
committer Jingning Han <jingning@google.com> Wed Jul 05 22:45:52 2017 +0000
tree 9b1ef267a1d8f59853447a5fecf2dd5830b597c2
parent 42595a75bef2d93be1d7234756d006c9501f0061

Support ext-tx in transform kernel selection

Support the selection from up to 16 transform types in the
transform kernel selection process.

Change-Id: Ia2acdf99a5dc9bcd4541989403c88a47443af7c6

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 853516a..fc7e8b9 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -1884,6 +1884,16 @@
       // if it's not, we skip it here.
       continue;
     }
+
+#if CONFIG_EXT_TX
+    int is_inter = is_inter_block(mbmi);
+    int ext_tx_set = get_ext_tx_set(get_min_tx_size(tx_size), mbmi->sb_type,
+                                    is_inter, cm->reduced_tx_set_used);
+    if (!(is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) &&
+        !(!is_inter && ext_tx_used_intra[ext_tx_set][tx_type]))
+      continue;
+#endif  // CONFIG_EXT_TX
+
     RD_STATS this_rd_stats;
     av1_invalid_rd_stats(&this_rd_stats);
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,