commit b1fcd1751e2b63efbb9f6b4960a42bd757e0c47e
author Alex Converse <aconverse@google.com> Thu Nov 19 14:53:51 2015 -0800
committer Alex Converse <aconverse@google.com> Thu Nov 19 15:00:59 2015 -0800
tree 0cfd845485f35c2ad1da2ff10f8c39c3671ff14e
parent f3f6b6fe3e960959489db2568d9942aeca261daa

Fix unsigned overflow in rd_variance_adjustment.

Found with clang -fsanitize=integer

Change-Id: I2538e7483cb2d5f06bceecbd3326bdd88bfecfa1

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 8ba2110..d87d164 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -3017,9 +3017,9 @@
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }
 

vp9/encoder/vp9_rdopt.c

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index ae14464..2a6b707 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2831,9 +2831,9 @@
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }