commit e61bff9b97344e1020ad2801fd2fed71091a0f3a
author Grant Hsu <grant.hsu@cidana.com> Mon Sep 10 10:02:09 2018 +0800
committer Grant Hsu <grant.hsu@cidana.com> Mon Sep 10 10:09:40 2018 +0800
tree 3145cb4fdafcf4af8ea299bb852f5dcb4d2c3ed4
parent 0cbad855c2a2cfbac8c7e61ad1ca8da85eec16b8

Fix overflow issue in get_est_rate_dist

Don't calculate est_residue_cost if sse is smaller than dist_mean.

BUG=aomedia:2139

Change-Id: I2752c977f69a4724588268d3f86f62d5bc74cc36

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index a0a60be..4c5f21a 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -646,11 +646,12 @@
   const InterModeRdModel *md = &tile_data->inter_mode_rd_models[bsize];
   if (md->ready) {
     const double est_ld = md->a * sse + md->b;
-    *est_residue_cost = (int)round((sse - md->dist_mean) / est_ld);
-    *est_dist = (int64_t)round(md->dist_mean);
     if (sse < md->dist_mean) {
       *est_residue_cost = 0;
       *est_dist = sse;
+    } else {
+      *est_residue_cost = (int)round((sse - md->dist_mean) / est_ld);
+      *est_dist = (int64_t)round(md->dist_mean);
     }
     return 1;
   }