commit e5fbdd39793e2617bd71205ea1f2fefa09ae0cd7
author Debargha Mukherjee <debargha@google.com> Tue Jan 08 12:55:43 2019 -0800
committer Debargha Mukherjee <debargha@google.com> Wed Jan 09 17:34:41 2019 +0000
tree e2ce8ffebdb5311c7416bacd837b4711c80623a8
parent 06e2c4fa33edb7f27702fbe2ad2311622da956b4

A fix in online inter_mode_rd_model_estimation

Change-Id: Ieed39ef1adfcf7b50e641aea414eb217d2a7f782

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 2d4c78e..a4c18a1a 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -776,10 +776,17 @@
       const double est_ld = md->a * sse + md->b;
       // Clamp estimated rate cost by INT_MAX / 2.
       // TODO(angiebird@google.com): find better solution than clamping.
-      *est_residue_cost =
-          fabs(est_ld) < 1e-2
-              ? INT_MAX / 2
-              : (int)AOMMIN(round((sse - md->dist_mean) / est_ld), INT_MAX / 2);
+      if (fabs(est_ld) < 1e-2) {
+        *est_residue_cost = INT_MAX / 2;
+      } else {
+        double est_residue_cost_dbl = ((sse - md->dist_mean) / est_ld);
+        if (est_residue_cost_dbl < 0) {
+          *est_residue_cost = 0;
+        } else {
+          *est_residue_cost =
+              (int)AOMMIN((int64_t)round(est_residue_cost_dbl), INT_MAX / 2);
+        }
+      }
       if (*est_residue_cost <= 0) {
         *est_residue_cost = 0;
         *est_dist = sse;