commit 1a5ab9f24ed9c19e0af5a757aed16654b1136b07
author Yue Chen <yuec@google.com> Thu Dec 14 14:53:51 2017 -0800
committer Yue Chen <yuec@google.com> Fri Dec 15 05:28:58 2017 +0000
tree 61b5de3e71373f04ff70892c08681516db5318a3
parent b971c8660298e2db643a9199692c201690ab7038

Avoid trying filter_intra + >32x32 tx

Filter intra mode is disabled if any dimention of tx_size is
greater than 32. In this patch, the max qualified tx_size is
found and set for testing filter_intra in intra_model_yrd().

BUG=aomedia:1058

Change-Id: I105aa68e722fce74b8828860a40040764cf20118

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 48e456f..caf1c56 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2721,7 +2721,13 @@
   RD_STATS this_rd_stats;
   int row, col;
   int64_t temp_sse, this_rd;
-  const TX_SIZE tx_size = tx_size_from_tx_mode(bsize, cpi->common.tx_mode, 0);
+  TX_SIZE tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 0);
+#if CONFIG_FILTER_INTRA
+  if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) {
+    tx_size = av1_max_tx_size_for_filter_intra(bsize, cm->tx_mode);
+    if (!av1_filter_intra_allowed_txsize(tx_size)) return INT64_MAX;
+  }
+#endif
   const int stepr = tx_size_high_unit[tx_size];
   const int stepc = tx_size_wide_unit[tx_size];
   const int max_blocks_wide = max_block_wide(xd, bsize, 0);