commit 95e13e23dc83dc151d52df49c60155e9af65c32e
author Yue Chen <yuec@google.com> Wed Nov 01 23:56:35 2017 -0700
committer Yue Chen <yuec@google.com> Fri Nov 03 22:02:14 2017 +0000
tree 9caeedcf80875bb9d76949677fe76ec7911514dc
parent 8315daf78e7f9782a9c3503c38c0ae7f1a38599b

Disable filter_intra mode in <8x8 tx blocks

0.159% gain on lowres 60 frames, compared to 0.236% gain if we don't
restrict it in small tx blocks.
(--disable-ext-partition --disable-ext-partition-types
 --disable-convolve-round --disable-ext-comp-refs)

Change-Id: I1d1c5474ca27de9dec992ea30a9883afd7a56474

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 4cd2388..17b3f6f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2466,6 +2466,12 @@
 
   mbmi->tx_type = tx_type;
   mbmi->tx_size = tx_size;
+#if CONFIG_FILTER_INTRA
+  if (!is_inter_block(mbmi) &&
+      mbmi->filter_intra_mode_info.use_filter_intra_mode[0] &&
+      !av1_filter_intra_allowed_txsize(tx_size))
+    return INT64_MAX;
+#endif
   txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, tx_size,
                    cpi->sf.use_fast_coef_costing);
   if (rd_stats->rate == INT_MAX) return INT64_MAX;
@@ -3285,7 +3291,7 @@
     int this_rate;
     int64_t this_rd, this_model_rd;
     RD_STATS tokenonly_rd_stats;
-    if (skip_mask & (1 << mode)) continue;
+    if (skip_mask & (1 << fimode_to_intradir[mode])) continue;
     mbmi->filter_intra_mode_info.filter_intra_mode[0] = mode;
     this_model_rd = intra_model_yrd(cpi, x, bsize, mode_cost);
     if (*best_model_rd != INT64_MAX &&
@@ -3735,7 +3741,7 @@
           x->palette_y_mode_cost[bsize - BLOCK_8X8][palette_y_mode_ctx][0];
     }
 #if CONFIG_FILTER_INTRA
-    if (mbmi->mode == DC_PRED)
+    if (mbmi->mode == DC_PRED && av1_filter_intra_allowed_txsize(mbmi->tx_size))
       this_rate += av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0);
 #endif  // CONFIG_FILTER_INTRA
 #if CONFIG_EXT_INTRA
@@ -3784,7 +3790,7 @@
   }
 
 #if CONFIG_FILTER_INTRA
-  if (beat_best_rd) {
+  if (beat_best_rd && av1_filter_intra_allowed_bsize(bsize)) {
     if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion,
                                  skippable, bsize, bmode_costs[DC_PRED],
                                  &best_rd, &best_model_rd,
@@ -10020,20 +10026,21 @@
       skippable = rd_stats_y.skip;
 
 #if CONFIG_FILTER_INTRA
-      int64_t best_rd_tmp = INT64_MAX;
-      if (rate_y != INT_MAX) {
-        best_rd_tmp = RDCOST(
-            x->rdmult,
-            rate_y + av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0) +
-                intra_mode_cost[mbmi->mode],
-            distortion_y);
-      }
       if (mbmi->mode == DC_PRED) {
         RD_STATS rd_stats_y_fi;
         int filter_intra_selected_flag = 0;
         TX_SIZE best_tx_size = mbmi->tx_size;
         TX_TYPE best_tx_type = mbmi->tx_type;
         FILTER_INTRA_MODE best_fi_mode = FILTER_DC_PRED;
+        int64_t best_rd_tmp = INT64_MAX;
+        if (rate_y != INT_MAX &&
+            av1_filter_intra_allowed_txsize(best_tx_size)) {
+          best_rd_tmp = RDCOST(
+              x->rdmult,
+              rate_y + av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0) +
+                  intra_mode_cost[mbmi->mode],
+              distortion_y);
+        }
 
         mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 1;
         for (FILTER_INTRA_MODE fi_mode = FILTER_DC_PRED;