commit cb51228307ab42e97a6cdb06862d57b51bb71273
author Jingning Han <jingning@google.com> Fri Feb 10 14:21:35 2017 -0800
committer Jingning Han <jingning@google.com> Sun Feb 12 20:08:51 2017 +0000
tree 6ed33c3eb7d03ddf77a0628ba307a1ed5904e0fd
parent 118bf67cb61f492023da085eb7446f6d9cbd598c

Separate intra tx_size logic between var-tx and rect-tx

Skip rectangular transform block size coding for intra coded block
in var-tx mode, when the rect-tx is disabled.

Change-Id: If3a091d25f19bf4a67485b5d235bb3d7d0c2cd03

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index b363417..ff28ac1 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -435,8 +435,12 @@
   const BLOCK_SIZE bsize = mbmi->sb_type;
 // For sub8x8 blocks the tx_size symbol does not need to be sent
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
   if (bsize > BLOCK_4X4) {
 #else
+  if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter_block(mbmi))) {
+#endif  // CONFIG_RECT_TX
+#else
   if (bsize >= BLOCK_8X8) {
 #endif
     const TX_SIZE tx_size = mbmi->tx_size;
@@ -446,7 +450,6 @@
                                      : intra_tx_size_cat_lookup[bsize];
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
     const int depth = tx_size_to_depth(coded_tx_size);
-
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
     assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
     assert(
@@ -1428,8 +1431,12 @@
 
   if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
       bsize > BLOCK_4X4 &&
 #else
+      (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) &&
+#endif  // CONFIG_RECT_TX
+#else
       bsize >= BLOCK_8X8 &&
 #endif
 #if CONFIG_SUPERTX
@@ -1795,9 +1802,13 @@
 
   if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
       bsize > BLOCK_4X4 &&
 #else
       bsize >= BLOCK_8X8 &&
+#endif  // CONFIG_RECT_TX
+#else
+      bsize >= BLOCK_8X8 &&
 #endif
       !xd->lossless[mbmi->segment_id])
     write_selected_tx_size(cm, xd, w);