commit f5f5ca03ca9b2a3c7299cafb653cebf0d2db71a2
author Alex Converse <aconverse@google.com> Mon May 09 22:16:22 2016 +0000
committer Gerrit Code Review <noreply-gerritcodereview@google.com> Mon May 09 22:16:22 2016 +0000
tree 6dea5c2a3b72d5323f5115d9c063d91c6f523621
parent c2c5297595dbd181a6aeda77800022297eb8c210
parent d8a18186ba4482ff121aa526fbf6df31f9d90d35

Merge "pickmode: Fix a pair of unsigned overflows."

vp9/encoder/vp9_pickmode.c

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 98fc603..d2ecafc 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -244,7 +244,7 @@
                     &sse8x8[k], &sum8x8[k]);
       *sse += sse8x8[k];
       *sum += sum8x8[k];
-      var8x8[k] = sse8x8[k] - (((unsigned int)sum8x8[k] * sum8x8[k]) >> 6);
+      var8x8[k] = sse8x8[k] - (uint32_t)(((int64_t)sum8x8[k] * sum8x8[k]) >> 6);
       k++;
     }
   }
@@ -265,7 +265,7 @@
           sse_i[(i + 1) * nw + j] + sse_i[(i + 1) * nw + j + 1];
       sum_o[k] = sum_i[i * nw + j] + sum_i[i * nw + j + 1] +
           sum_i[(i + 1) * nw + j] + sum_i[(i + 1) * nw + j + 1];
-      var_o[k] = sse_o[k] - (((unsigned int)sum_o[k] * sum_o[k]) >>
+      var_o[k] = sse_o[k] - (uint32_t)(((int64_t)sum_o[k] * sum_o[k]) >>
           (b_width_log2_lookup[unit_size] +
               b_height_log2_lookup[unit_size] + 6));
       k++;