commit 1a5ab9f24ed9c19e0af5a757aed16654b1136b07
author Yue Chen <yuec@google.com> Thu Dec 14 14:53:51 2017 -0800
committer Yue Chen <yuec@google.com> Fri Dec 15 05:28:58 2017 +0000
tree 61b5de3e71373f04ff70892c08681516db5318a3
parent b971c8660298e2db643a9199692c201690ab7038

Avoid trying filter_intra + >32x32 tx

Filter intra mode is disabled if any dimention of tx_size is
greater than 32. In this patch, the max qualified tx_size is
found and set for testing filter_intra in intra_model_yrd().

BUG=aomedia:1058

Change-Id: I105aa68e722fce74b8828860a40040764cf20118

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 2eb1a52..96bc17b 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -195,7 +195,6 @@
 } PALETTE_MODE_INFO;
 
 #if CONFIG_FILTER_INTRA
-#define USE_3TAP_INTRA_FILTER 1  // 0: 4-tap; 1: 3-tap
 typedef struct {
   // 1: an ext intra mode is used; 0: otherwise.
   uint8_t use_filter_intra_mode[PLANE_TYPES];
@@ -205,27 +204,6 @@
 static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = {
   DC_PRED, V_PRED, H_PRED, D153_PRED, DC_PRED
 };
-
-#define DISABLE_SUB8X8_FILTER_INTRA 0
-
-static INLINE int av1_filter_intra_allowed_bsize(BLOCK_SIZE bs) {
-  (void)bs;
-#if DISABLE_SUB8X8_FILTER_INTRA
-  return block_size_wide[bs] >= 8 && block_size_high[bs] >= 8;
-#else
-  return 1;
-#endif
-}
-
-static INLINE int av1_filter_intra_allowed_txsize(TX_SIZE tx) {
-  (void)tx;
-#if DISABLE_SUB8X8_FILTER_INTRA
-  return tx_size_wide[tx] >= 8 && tx_size_high[tx] >= 8 &&
-         tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
-#else
-  return tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
-#endif
-}
 #endif  // CONFIG_FILTER_INTRA
 
 #if CONFIG_RD_DEBUG
@@ -982,6 +960,47 @@
 };
 #endif  // CONFIG_EXT_INTRA
 
+#if CONFIG_FILTER_INTRA
+#define DISABLE_SUB8X8_FILTER_INTRA 0
+
+static INLINE int av1_filter_intra_allowed_bsize(BLOCK_SIZE bs) {
+  (void)bs;
+#if DISABLE_SUB8X8_FILTER_INTRA
+  return block_size_wide[bs] >= 8 && block_size_high[bs] >= 8;
+#else
+  return 1;
+#endif
+}
+
+static INLINE int av1_filter_intra_allowed_txsize(TX_SIZE tx) {
+  (void)tx;
+  if (tx == TX_INVALID) return 0;
+
+#if DISABLE_SUB8X8_FILTER_INTRA
+  return tx_size_wide[tx] >= 8 && tx_size_high[tx] >= 8 &&
+         tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
+#else
+  return tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
+#endif
+}
+
+static INLINE TX_SIZE av1_max_tx_size_for_filter_intra(BLOCK_SIZE bsize,
+                                                       TX_MODE tx_mode) {
+  const TX_SIZE max_tx_size = tx_size_from_tx_mode(bsize, tx_mode, 0);
+
+  if (tx_mode != TX_MODE_SELECT) return max_tx_size;
+
+  int depth = 0;
+  TX_SIZE tx_size = max_tx_size;
+  while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
+    if (av1_filter_intra_allowed_txsize(tx_size)) return tx_size;
+    depth++;
+    tx_size = sub_tx_size_map[0][tx_size];
+  }
+  return TX_INVALID;
+}
+#endif
+
 #if CONFIG_DCT_ONLY
 #define FIXED_TX_TYPE 1
 #else