commit 18f6c15cfa71a8ef5b3a40f6b9b85e89b9c67381
author Yue Chen <yuec@google.com> Mon Nov 06 11:23:47 2017 -0800
committer Yue Chen <yuec@google.com> Tue Nov 07 01:13:05 2017 +0000
tree 818d4a74c05d96ea49e31766f2973cb4fc5b44ea
parent 921b334f3f79fe5f2d23e2e1665878c781d2649a

Fix mismatches caused by filter_intra

Return invalid rate (previously only invalid rdcost) if the
mode combination to check is < 8x8 tx_size + filter_intra mode.

BUG=aomedia:1006

Change-Id: If90f431c7692473c88ac7a644bfa969a1acb3573

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 92249a1..4a483b3 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2440,8 +2440,10 @@
 #if CONFIG_FILTER_INTRA
   if (!is_inter_block(mbmi) &&
       mbmi->filter_intra_mode_info.use_filter_intra_mode[0] &&
-      !av1_filter_intra_allowed_txsize(tx_size))
+      !av1_filter_intra_allowed_txsize(tx_size)) {
+    rd_stats->rate = INT_MAX;
     return INT64_MAX;
+  }
 #endif
   txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, tx_size,
                    cpi->sf.use_fast_coef_costing);
@@ -3134,7 +3136,7 @@
   MB_MODE_INFO *mbmi = &mic->mbmi;
   int filter_intra_selected_flag = 0;
   FILTER_INTRA_MODE mode;
-  TX_SIZE best_tx_size = TX_4X4;
+  TX_SIZE best_tx_size = TX_8X8;
   FILTER_INTRA_MODE_INFO filter_intra_mode_info;
   TX_TYPE best_tx_type;
 
@@ -3619,7 +3621,8 @@
   }
 
 #if CONFIG_FILTER_INTRA
-  if (beat_best_rd && av1_filter_intra_allowed_bsize(bsize)) {
+  if (beat_best_rd && av1_filter_intra_allowed_bsize(bsize) &&
+      !xd->lossless[mbmi->segment_id]) {
     if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion,
                                  skippable, bsize, bmode_costs[DC_PRED],
                                  &best_rd, &best_model_rd,
@@ -9746,7 +9749,8 @@
       skippable = rd_stats_y.skip;
 
 #if CONFIG_FILTER_INTRA
-      if (mbmi->mode == DC_PRED) {
+      if (mbmi->mode == DC_PRED && !xd->lossless[mbmi->segment_id] &&
+          av1_filter_intra_allowed_bsize(mbmi->sb_type)) {
         RD_STATS rd_stats_y_fi;
         int filter_intra_selected_flag = 0;
         TX_SIZE best_tx_size = mbmi->tx_size;