commit 1a16a987eee60d089bab7a47d25320c48762cf8c
author Debargha Mukherjee <debargha@google.com> Tue Oct 04 11:50:00 2016 -0700
committer Debargha Mukherjee <debargha@google.com> Tue Oct 04 11:50:00 2016 -0700
tree 382581303585fe0569fbe67c1eb99a244246109e
parent 438b1dcb729dd52e435a83e0f91812f9d3c4568c

Fix an integer overflow issue in restoration

https://bugs.chromium.org/p/webm/issues/detail?id=1306

Change-Id: Icd11d373ff08954121c097728e4c7791791e223f

av1/common/restoration.c

diff --git a/av1/common/restoration.c b/av1/common/restoration.c
index 97c89e1..8f0a212 100644
--- a/av1/common/restoration.c
+++ b/av1/common/restoration.c
@@ -218,15 +218,15 @@
 
     for (i = 0; i < (v_end - v_start); ++i) {
       for (j = 0; j < (h_end - h_start); ++j) {
-        int x, y;
-        int flsum = 0, wtsum = 0, wt;
+        int x, y, wt;
+        int64_t flsum = 0, wtsum = 0;
         uint8_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride;
         for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
           for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
             wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
                  (int)wr_lut_[data_p2[x] - data_p[j]];
-            wtsum += wt;
-            flsum += wt * data_p2[x];
+            wtsum += (int64_t)wt;
+            flsum += (int64_t)wt * data_p2[x];
           }
           data_p2 += stride;
         }
@@ -392,15 +392,15 @@
 
     for (i = 0; i < (v_end - v_start); ++i) {
       for (j = 0; j < (h_end - h_start); ++j) {
-        int x, y;
-        int flsum = 0, wtsum = 0, wt;
+        int x, y, wt;
+        int64_t flsum = 0, wtsum = 0;
         uint16_t *data_p2 = data_p + j - RESTORATION_HALFWIN * stride;
         for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
           for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
             wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
                  (int)wr_lut_[data_p2[x] - data_p[j]];
-            wtsum += wt;
-            flsum += wt * data_p2[x];
+            wtsum += (int64_t)wt;
+            flsum += (int64_t)wt * data_p2[x];
           }
           data_p2 += stride;
         }