commit 897dd193c5dc853fcd0f65e2e4a8bc9abb5ca8fd
author Yaowu Xu <yaowu@google.com> Wed Aug 02 18:41:56 2017 -0700
committer Yaowu Xu <yaowu@google.com> Thu Aug 03 21:35:32 2017 +0000
tree 3f41906a3536ed3810e3e3b4cf05c221d899ff5f
parent 708e08b74a89bf290dbc1b8338cba55abea406bc

Avoid the use of INT_MAX in rate computation

BUG=aomedia:676

Change-Id: I278e84de26d995419b85039f7b78b8324cadf3a3

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index f7432e0..b6a2637 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -3558,10 +3558,13 @@
                      bsize, ctx_none, best_rdc.rdcost);
     if (this_rdc.rate != INT_MAX) {
       if (bsize_at_least_8x8) {
-        this_rdc.rate += partition_cost[PARTITION_NONE];
+        const int pt_cost = partition_cost[PARTITION_NONE] < INT_MAX
+                                ? partition_cost[PARTITION_NONE]
+                                : 0;
+        this_rdc.rate += pt_cost;
         this_rdc.rdcost = RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist);
 #if CONFIG_SUPERTX
-        this_rate_nocoef += partition_cost[PARTITION_NONE];
+        this_rate_nocoef += pt_cost;
 #endif
       }