commit c9c78b500d1947ca3608dadf492e5c71f45c7967
author Hui Su <huisu@google.com> Fri Nov 17 14:21:32 2017 -0800
committer Hui Su <huisu@google.com> Thu Nov 23 04:49:52 2017 +0000
tree c594948f2281b2242716c280c46dce05367d3c60
parent a60e26d594ff97b1477cf3079ad7ceafd4901ed4

Avoid interger overflow in aom_blend_*_c()

BUG=b/69445855,b/69441422

Change-Id: Iaf5aba78dc39f01c87fb726611e674d34af6bffe
(cherry picked from commit 75ff22f309de2e25477d336a6a8e9e58d3bb2272)

aom_dsp/blend_a64_mask.c

diff --git a/aom_dsp/blend_a64_mask.c b/aom_dsp/blend_a64_mask.c
index c35fa19..5cd3dac 100644
--- a/aom_dsp/blend_a64_mask.c
+++ b/aom_dsp/blend_a64_mask.c
@@ -41,40 +41,40 @@
   if (subw == 0 && subh == 0) {
     for (i = 0; i < h; ++i) {
       for (j = 0; j < w; ++j) {
-        const int m = mask[i * mask_stride + j];
-        dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j],
-                                                src1[i * src1_stride + j]);
+        const int64_t m = mask[i * mask_stride + j];
+        dst[i * dst_stride + j] = (int32_t)AOM_BLEND_A64(
+            m, src0[i * src0_stride + j], src1[i * src1_stride + j]);
       }
     }
   } else if (subw == 1 && subh == 1) {
     for (i = 0; i < h; ++i) {
       for (j = 0; j < w; ++j) {
-        const int m = ROUND_POWER_OF_TWO(
+        const int64_t m = ROUND_POWER_OF_TWO(
             mask[(2 * i) * mask_stride + (2 * j)] +
                 mask[(2 * i + 1) * mask_stride + (2 * j)] +
                 mask[(2 * i) * mask_stride + (2 * j + 1)] +
                 mask[(2 * i + 1) * mask_stride + (2 * j + 1)],
             2);
-        dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j],
-                                                src1[i * src1_stride + j]);
+        dst[i * dst_stride + j] = (int32_t)AOM_BLEND_A64(
+            m, src0[i * src0_stride + j], src1[i * src1_stride + j]);
       }
     }
   } else if (subw == 1 && subh == 0) {
     for (i = 0; i < h; ++i) {
       for (j = 0; j < w; ++j) {
-        const int m = AOM_BLEND_AVG(mask[i * mask_stride + (2 * j)],
-                                    mask[i * mask_stride + (2 * j + 1)]);
-        dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j],
-                                                src1[i * src1_stride + j]);
+        const int64_t m = AOM_BLEND_AVG(mask[i * mask_stride + (2 * j)],
+                                        mask[i * mask_stride + (2 * j + 1)]);
+        dst[i * dst_stride + j] = (int32_t)AOM_BLEND_A64(
+            m, src0[i * src0_stride + j], src1[i * src1_stride + j]);
       }
     }
   } else {
     for (i = 0; i < h; ++i) {
       for (j = 0; j < w; ++j) {
-        const int m = AOM_BLEND_AVG(mask[(2 * i) * mask_stride + j],
-                                    mask[(2 * i + 1) * mask_stride + j]);
-        dst[i * dst_stride + j] = AOM_BLEND_A64(m, src0[i * src0_stride + j],
-                                                src1[i * src1_stride + j]);
+        const int64_t m = AOM_BLEND_AVG(mask[(2 * i) * mask_stride + j],
+                                        mask[(2 * i + 1) * mask_stride + j]);
+        dst[i * dst_stride + j] = (int32_t)AOM_BLEND_A64(
+            m, src0[i * src0_stride + j], src1[i * src1_stride + j]);
       }
     }
   }