commit 7ebfaf4f5a5c7297abf622bf3713805f87f61b6c
author Yue Chen <yuec@google.com> Tue Jan 16 10:40:38 2018 -0800
committer Yaowu Xu <yaowu@google.com> Wed Jan 17 02:17:05 2018 +0000
tree 951af293cb99f5d4022829257f5890f40676cf4d
parent 70281e67f3008a3356a62c0f2e539859536acd5b

Fix data overflow in interintra rd

BUG=aomedia:1236

Change-Id: I72c1e5ba7f0090c0831ca9aaccbc32dc4f1fdf4f

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 444135a..59725d0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -7547,7 +7547,7 @@
         rd = RDCOST(x->rdmult, rate_mv + rmode + rate_sum, dist_sum);
       best_interintra_rd = rd;
 
-      if (ref_best_rd < INT64_MAX && best_interintra_rd > 2 * ref_best_rd)
+      if (ref_best_rd < INT64_MAX && (best_interintra_rd >> 1) > ref_best_rd)
         continue;
 
       if (is_interintra_wedge_used(bsize)) {
@@ -7619,7 +7619,7 @@
           if (best_interintra_rd_wedge < best_interintra_rd_nowedge) {
             mbmi->use_wedge_interintra = 1;
             mbmi->mv[0].as_int = tmp_mv.as_int;
-            rd_stats->rate += tmp_rate_mv - rate_mv;
+            tmp_rate2 += tmp_rate_mv - rate_mv;
             rate_mv = tmp_rate_mv;
           } else {
             mbmi->use_wedge_interintra = 0;