commit 9deaa07c75161d4e918f2d79a739675d779b6177
author Yue Chen <yuec@google.com> Tue May 03 21:27:21 2022 +0000
committer Leo (Liang) Zhao <leolzhao@tencent.com> Tue May 03 21:27:21 2022 +0000
tree eaed0ab83f15495e3375c9cfa04e51dee1355525
parent 8734d810ef261b93663e11aa7b0091f73b48905c

Fix an infinite loop in NEW_TX_PARTITION + IST

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 95a8c98..26279fa 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1524,6 +1524,7 @@
   return ctx_size == tx_size;
 }
 
+#if !CONFIG_NEW_TX_PARTITION
 static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
   TX_SIZE ctx_size = max_txsize_rect_lookup[bsize];
   int depth = 0;
@@ -1533,6 +1534,8 @@
   }
   return depth;
 }
+#endif
+
 /*
  * If secondary transform is enabled (CONFIG_IST) :
  * Bits 4~5 of tx_type stores secondary tx_type
@@ -1593,11 +1596,11 @@
 #endif  // CONFIG_IST_FIX_B076
     ist_eob = 0;
   }
-  const int depth = tx_size_to_depth(tx_size, bs);
+  const int is_depth0 = tx_size_is_depth0(tx_size, bs);
   const int code_stx =
       (primary_tx_type == DCT_DCT || primary_tx_type == ADST_ADST) &&
       (intra_dir < PAETH_PRED) &&
-      !(mbmi->filter_intra_mode_info.use_filter_intra) && !(depth) && ist_eob;
+      !(mbmi->filter_intra_mode_info.use_filter_intra) && is_depth0 && ist_eob;
   return code_stx;
 }
 #endif