commit 8f699e057cd1ad30bfb05db5d0029701b4a0fde6
author Yaowu Xu <yaowu@google.com> Thu Dec 28 14:34:48 2017 -0800
committer Yaowu Xu <yaowu@google.com> Sat Dec 30 06:13:05 2017 +0000
tree 6d0f91d3e91461d31c03b00e3773a9ed3f5142e0
parent 92a890d167eb02321af4f3c77ff3542b5469ce39

Prevent overflow when distortion is INT64_MAX

Change-Id: Ic2d7026b5a859666e813766b34badd4a9ee209b7

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index cf3f118..38d94ff 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -10393,8 +10393,11 @@
 
       // Compare the use of skip_mode with the best intra/inter mode obtained.
       const int64_t best_intra_inter_mode_cost =
-          RDCOST(x->rdmult, rd_cost->rate + x->skip_mode_cost[skip_mode_ctx][0],
-                 rd_cost->dist);
+          (rd_cost->dist < INT64_MAX && rd_cost->rate < INT32_MAX)
+              ? RDCOST(x->rdmult,
+                       rd_cost->rate + x->skip_mode_cost[skip_mode_ctx][0],
+                       rd_cost->dist)
+              : INT64_MAX;
 
       if (x->skip_mode_rdcost <= best_intra_inter_mode_cost)
         best_mbmode.skip_mode = 1;