commit 2176b7acc28a651ceb6a1a202cd82bf0f4fa6874
author Alex Converse <aconverse@google.com> Thu Jul 28 09:48:50 2016 -0700
committer James Zern <jzern@google.com> Thu Oct 13 20:12:20 2016 -0700
tree f1a72ac36ecb894d3c428c0dc133668115a5e028
parent 078856a4df0415dae2f6d3d9a7bea52320dd565e

Resolve -Wshorten-64-to-32 in variance.

The subtrahend is small enough to fit into uint32_t.

Change-Id: Ic4d7128aaa665eaf6b25d562610ba8942c46137f
(cherry picked from commit c0241664aac3a1805db9bd8e09e071ac326531e0)

aom_dsp/variance.c

diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index 90617d9..c93f98e 100644
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -148,7 +148,7 @@
                                      uint32_t *sse) {                \
     int sum;                                                         \
     variance(a, a_stride, b, b_stride, W, H, sse, &sum);             \
-    return *sse - (((int64_t)sum * sum) / (W * H));                  \
+    return *sse - (uint32_t)(((int64_t)sum * sum) / (W * H));        \
   }
 
 #define SUBPIX_VAR(W, H)                                                \

aom_dsp/x86/variance_sse2.c

diff --git a/aom_dsp/x86/variance_sse2.c b/aom_dsp/x86/variance_sse2.c
index 0466837..d9563aa 100644
--- a/aom_dsp/x86/variance_sse2.c
+++ b/aom_dsp/x86/variance_sse2.c
@@ -249,7 +249,7 @@
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 32 * 32);
   assert(sum >= -255 * 32 * 32);
-  return *sse - (((int64_t)sum * sum) >> 10);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 10);
 }
 
 unsigned int aom_variance32x16_sse2(const uint8_t *src, int src_stride,
@@ -260,7 +260,7 @@
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 32 * 16);
   assert(sum >= -255 * 32 * 16);
-  return *sse - (((int64_t)sum * sum) >> 9);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 9);
 }
 
 unsigned int aom_variance16x32_sse2(const uint8_t *src, int src_stride,
@@ -271,7 +271,7 @@
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 32 * 16);
   assert(sum >= -255 * 32 * 16);
-  return *sse - (((int64_t)sum * sum) >> 9);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 9);
 }
 
 unsigned int aom_variance64x64_sse2(const uint8_t *src, int src_stride,
@@ -282,7 +282,7 @@
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 64 * 64);
   assert(sum >= -255 * 64 * 64);
-  return *sse - (((int64_t)sum * sum) >> 12);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 12);
 }
 
 unsigned int aom_variance64x32_sse2(const uint8_t *src, int src_stride,
@@ -293,7 +293,7 @@
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 64 * 32);
   assert(sum >= -255 * 64 * 32);
-  return *sse - (((int64_t)sum * sum) >> 11);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 11);
 }
 
 unsigned int aom_variance32x64_sse2(const uint8_t *src, int src_stride,
@@ -304,7 +304,7 @@
                 aom_get16x16var_sse2, 16);
   assert(sum <= 255 * 64 * 32);
   assert(sum >= -255 * 64 * 32);
-  return *sse - (((int64_t)sum * sum) >> 11);
+  return *sse - (unsigned int)(((int64_t)sum * sum) >> 11);
 }
 
 unsigned int aom_mse8x8_sse2(const uint8_t *src, int src_stride,
@@ -381,7 +381,7 @@
       }                                                                        \
     }                                                                          \
     *sse_ptr = sse;                                                            \
-    return sse - (cast_prod(cast se * se) >> (wlog2 + hlog2));                 \
+    return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2));   \
   }
 
 #define FNS(opt1, opt2)                              \
@@ -452,7 +452,7 @@
       }                                                                        \
     }                                                                          \
     *sseptr = sse;                                                             \
-    return sse - (cast_prod(cast se * se) >> (wlog2 + hlog2));                 \
+    return sse - (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2));   \
   }
 
 #define FNS(opt1, opt2)                              \