commit b93880aafb882acdd2dfba86c24343fe385d8b89
author Hui Su <huisu@google.com> Thu Jun 27 18:29:30 2019 -0700
committer Hui Su <huisu@google.com> Mon Jul 08 16:06:30 2019 +0000
tree 445da7a4bfd207f2c148878cbf01379b15ec5835
parent e2c1d5ef82e86db553ae8112fc68c6cefee1f8d0

Fix RD calculation in CDEF search

Use the correct rdmult.
About 0.05% coding gain when tested on speed 1 30 frames.

STATS_CHANGED

Change-Id: I542a99e6dc22e9ffacf0863c7b50105538e509e5

av1/common/cdef.h

diff --git a/av1/common/cdef.h b/av1/common/cdef.h
index 70bdeaf..3848583 100644
--- a/av1/common/cdef.h
+++ b/av1/common/cdef.h
@@ -42,7 +42,8 @@
 void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, MACROBLOCKD *xd);
 
 void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
-                     AV1_COMMON *cm, MACROBLOCKD *xd, int pick_method);
+                     AV1_COMMON *cm, MACROBLOCKD *xd, int pick_method,
+                     int rdmult);
 
 #ifdef __cplusplus
 }  // extern "C"

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index c465e18..271946f 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -4226,7 +4226,7 @@
 #endif
     // Find CDEF parameters
     av1_cdef_search(&cm->cur_frame->buf, cpi->source, cm, xd,
-                    cpi->sf.cdef_pick_method);
+                    cpi->sf.cdef_pick_method, cpi->td.mb.rdmult);
 
     // Apply the filter
     av1_cdef_frame(&cm->cur_frame->buf, cm, xd);

av1/encoder/pickcdef.c

diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c
index ecadb0d..c23d9a9 100644
--- a/av1/encoder/pickcdef.c
+++ b/av1/encoder/pickcdef.c
@@ -359,7 +359,8 @@
 }
 
 void av1_cdef_search(YV12_BUFFER_CONFIG *frame, const YV12_BUFFER_CONFIG *ref,
-                     AV1_COMMON *cm, MACROBLOCKD *xd, int pick_method) {
+                     AV1_COMMON *cm, MACROBLOCKD *xd, int pick_method,
+                     int rdmult) {
   if (pick_method == CDEF_PICK_FROM_Q) {
     pick_cdef_from_qp(cm);
     return;
@@ -524,12 +525,7 @@
 
   /* Search for different number of signalling bits. */
   int nb_strength_bits = 0;
-  uint64_t best_tot_mse = UINT64_MAX;
-  const int quantizer =
-      av1_ac_quant_QTX(cm->base_qindex, 0, cm->seq_params.bit_depth) >>
-      (cm->seq_params.bit_depth - 8);
-  aom_clear_system_state();
-  const double lambda = .12 * quantizer * quantizer / 256.;
+  uint64_t best_rd = UINT64_MAX;
   CdefInfo *const cdef_info = &cm->cdef_info;
   for (int i = 0; i <= 3; i++) {
     int best_lev0[CDEF_MAX_STRENGTHS];
@@ -543,13 +539,14 @@
       tot_mse = joint_strength_search(best_lev0, nb_strengths, mse[0], sb_count,
                                       fast);
     }
-    /* Count superblock signalling cost. */
-    tot_mse += (uint64_t)(sb_count * lambda * i);
-    /* Count header signalling cost. */
-    tot_mse += (uint64_t)(nb_strengths * lambda * CDEF_STRENGTH_BITS *
-                          (num_planes > 1 ? 2 : 1));
-    if (tot_mse < best_tot_mse) {
-      best_tot_mse = tot_mse;
+
+    const int total_bits = sb_count * i + nb_strengths * CDEF_STRENGTH_BITS *
+                                              (num_planes > 1 ? 2 : 1);
+    const int rate_cost = av1_cost_literal(total_bits);
+    const uint64_t dist = tot_mse * 16;
+    const uint64_t rd = RDCOST(rdmult, rate_cost, dist);
+    if (rd < best_rd) {
+      best_rd = rd;
       nb_strength_bits = i;
       memcpy(cdef_info->cdef_strengths, best_lev0,
              nb_strengths * sizeof(best_lev0[0]));