commit 4e8048a0869388f8a4ffe79e1d901a01775d98ce
author Hui Su <huisu@google.com> Wed May 16 14:28:06 2018 -0700
committer Hui Su <huisu@google.com> Thu May 17 22:34:14 2018 +0000
tree 8fa3facb6bc48a5b0f40a35e4031fd3319f72169
parent f481d98fd7878b086aedf9b75bdedb9729db2b5d

Fix integer overflow in get_mean_and_dev()

BUG=aomedia:1889

Change-Id: Ib7c38689c646563129696b30553ee35a2cead3d5

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index a01bd3a..5c6d6c9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4044,7 +4044,7 @@
 static void get_mean_and_dev(const int16_t *data, int stride, int bw, int bh,
                              float *mean, float *dev) {
   int x_sum = 0;
-  int x2_sum = 0;
+  uint64_t x2_sum = 0;
   for (int i = 0; i < bh; ++i) {
     for (int j = 0; j < bw; ++j) {
       const int val = data[j];
@@ -4056,7 +4056,7 @@
 
   const int num = bw * bh;
   const float e_x = (float)x_sum / num;
-  const float e_x2 = (float)x2_sum / num;
+  const float e_x2 = (float)((double)x2_sum / num);
   const float diff = e_x2 - e_x * e_x;
   *dev = (diff > 0) ? sqrtf(diff) : 0;
   *mean = e_x;