commit cb51228307ab42e97a6cdb06862d57b51bb71273
author Jingning Han <jingning@google.com> Fri Feb 10 14:21:35 2017 -0800
committer Jingning Han <jingning@google.com> Sun Feb 12 20:08:51 2017 +0000
tree 6ed33c3eb7d03ddf77a0628ba307a1ed5904e0fd
parent 118bf67cb61f492023da085eb7446f6d9cbd598c

Separate intra tx_size logic between var-tx and rect-tx

Skip rectangular transform block size coding for intra coded block
in var-tx mode, when the rect-tx is disabled.

Change-Id: If3a091d25f19bf4a67485b5d235bb3d7d0c2cd03

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index bec0423..32463d4 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -420,8 +420,12 @@
   const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
   if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
   if (bsize > BLOCK_4X4) {
 #else
+  if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) {
+#endif  // CONFIG_RECT_TX
+#else
   if (bsize >= BLOCK_8X8) {
 #endif  // CONFIG_CB4X4 && CONFIG_VAR_TX
     if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) {

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index b363417..ff28ac1 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -435,8 +435,12 @@
   const BLOCK_SIZE bsize = mbmi->sb_type;
 // For sub8x8 blocks the tx_size symbol does not need to be sent
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
   if (bsize > BLOCK_4X4) {
 #else
+  if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter_block(mbmi))) {
+#endif  // CONFIG_RECT_TX
+#else
   if (bsize >= BLOCK_8X8) {
 #endif
     const TX_SIZE tx_size = mbmi->tx_size;
@@ -446,7 +450,6 @@
                                      : intra_tx_size_cat_lookup[bsize];
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
     const int depth = tx_size_to_depth(coded_tx_size);
-
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
     assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
     assert(
@@ -1428,8 +1431,12 @@
 
   if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
       bsize > BLOCK_4X4 &&
 #else
+      (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) &&
+#endif  // CONFIG_RECT_TX
+#else
       bsize >= BLOCK_8X8 &&
 #endif
 #if CONFIG_SUPERTX
@@ -1795,9 +1802,13 @@
 
   if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
       bsize > BLOCK_4X4 &&
 #else
       bsize >= BLOCK_8X8 &&
+#endif  // CONFIG_RECT_TX
+#else
+      bsize >= BLOCK_8X8 &&
 #endif
       !xd->lossless[mbmi->segment_id])
     write_selected_tx_size(cm, xd, w);

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 25afeda..7917b19 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5729,8 +5729,13 @@
 #endif
     if (cm->tx_mode == TX_MODE_SELECT &&
 #if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
+#if CONFIG_RECT_TX
         mbmi->sb_type > BLOCK_4X4 &&
 #else
+        (mbmi->sb_type >= BLOCK_8X8 ||
+         (mbmi->sb_type > BLOCK_4X4 && is_inter)) &&
+#endif
+#else
         mbmi->sb_type >= BLOCK_8X8 &&
 #endif
         !(is_inter && (mbmi->skip || seg_skip))) {