commit cb51228307ab42e97a6cdb06862d57b51bb71273
author Jingning Han <jingning@google.com> Fri Feb 10 14:21:35 2017 -0800
committer Jingning Han <jingning@google.com> Sun Feb 12 20:08:51 2017 +0000
tree 6ed33c3eb7d03ddf77a0628ba307a1ed5904e0fd
parent 118bf67cb61f492023da085eb7446f6d9cbd598c

Separate intra tx_size logic between var-tx and rect-tx

Skip rectangular transform block size coding for intra coded block
in var-tx mode, when the rect-tx is disabled.

Change-Id: If3a091d25f19bf4a67485b5d235bb3d7d0c2cd03

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 25afeda..7917b19 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5729,8 +5729,13 @@
 #endif
     if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
         mbmi->sb_type > BLOCK_4X4 &&
 #else
+        (mbmi->sb_type >= BLOCK_8X8 ||
+         (mbmi->sb_type > BLOCK_4X4 && is_inter)) &&
+#endif
+#else
         mbmi->sb_type >= BLOCK_8X8 &&
 #endif
         !(is_inter && (mbmi->skip || seg_skip))) {