commit 9ea1f64836445ca29926af00693c577db64d103a
author Yunqing Wang <yunqingwang@google.com> Tue Apr 06 23:48:01 2021 -0700
committer Yunqing Wang <yunqingwang@google.com> Thu Apr 15 22:51:57 2021 +0000
tree e947d60bb55cb2a3709330952572359356684550
parent 968fd688641fdd2eed23aeae1254d087fd3820a3

Rate cost calculation fixing in filter intra search

While use_filter_intra is 0, rate cost calculation is modified
to call intra_mode_info_cost_y(), which is made consistent with
rate cost calculation in filter intra search (handle_filter_
intra_mode()). This slightly changes rate cost in few cases
since intra_mode_info_cost_y() includes more cost (e.g. the case
when palette mode is also allowed). Borg test at speed 1 didn't
show much BDRate changes.
               avg_psnr: ovr_psnr: ssim:
midres2:        0.002     0.003    0.004
screen content: 0.009     0.023    0.014

STATS_CHANGED

Change-Id: If0e40e56d343dca3131dbbc5dedff87552a5dbf9

av1/encoder/intra_mode_search.c

diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index e5df7e5..f903731 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -1020,6 +1020,7 @@
   }
 }
 
+// Evaluate a given luma intra-mode in inter frames.
 int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state,
                             const AV1_COMP *cpi, MACROBLOCK *x,
                             BLOCK_SIZE bsize, unsigned int ref_frame_cost,
@@ -1082,8 +1083,12 @@
     int try_filter_intra = 1;
     int64_t best_rd_so_far = INT64_MAX;
     if (rd_stats_y->rate != INT_MAX) {
-      const int tmp_rate = rd_stats_y->rate +
-                           mode_costs->filter_intra_cost[bsize][0] + mode_cost;
+      // best_rd_so_far is the rdcost of DC_PRED without using filter_intra.
+      // Later, in filter intra search, best_rd_so_far is used for comparison.
+      mbmi->filter_intra_mode_info.use_filter_intra = 0;
+      const int tmp_rate =
+          rd_stats_y->rate +
+          intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost);
       best_rd_so_far = RDCOST(x->rdmult, tmp_rate, rd_stats_y->dist);
       try_filter_intra = (best_rd_so_far / 2) <= best_rd;
     }