commit fbaf7c53d00877dd7d70a04f02389a07a105ca49
author Aasaipriya Chandran <aasaipriya.c@ittiam.com> Mon Aug 24 13:07:24 2020 +0530
committer Yunqing Wang <yunqingwang@google.com> Thu Aug 27 20:24:02 2020 +0000
tree 726e2ab3326b43575ba30c4f9ca2e721b2f621be
parent 08e6af14d39e0c56f899d708866575cbe29b4b14

Fix integer overflow in 12bit SSIM calculation

This patch fixes an integer overflow issue in similarity()
function for SSIM calculation when input-bit-depth=12.

Change-Id: Id881a95c28d61a193c4ae22ab8d1fa737c75197f

aom_dsp/ssim.c

diff --git a/aom_dsp/ssim.c b/aom_dsp/ssim.c
index 95b8888..dd4e0eb 100644
--- a/aom_dsp/ssim.c
+++ b/aom_dsp/ssim.c
@@ -77,7 +77,7 @@
 static double similarity(uint32_t sum_s, uint32_t sum_r, uint32_t sum_sq_s,
                          uint32_t sum_sq_r, uint32_t sum_sxr, int count,
                          uint32_t bd) {
-  int64_t ssim_n, ssim_d;
+  double ssim_n, ssim_d;
   int64_t c1, c2;
   if (bd == 8) {
     // scale the constants by number of pixels
@@ -94,14 +94,14 @@
     assert(0);
   }
 
-  ssim_n = (2 * sum_s * sum_r + c1) *
-           ((int64_t)2 * count * sum_sxr - (int64_t)2 * sum_s * sum_r + c2);
+  ssim_n = (2.0 * sum_s * sum_r + c1) *
+           (2.0 * count * sum_sxr - 2.0 * sum_s * sum_r + c2);
 
-  ssim_d = (sum_s * sum_s + sum_r * sum_r + c1) *
-           ((int64_t)count * sum_sq_s - (int64_t)sum_s * sum_s +
-            (int64_t)count * sum_sq_r - (int64_t)sum_r * sum_r + c2);
+  ssim_d = ((double)sum_s * sum_s + (double)sum_r * sum_r + c1) *
+           ((double)count * sum_sq_s - (double)sum_s * sum_s +
+            (double)count * sum_sq_r - (double)sum_r * sum_r + c2);
 
-  return ssim_n * 1.0 / ssim_d;
+  return ssim_n / ssim_d;
 }
 
 static double ssim_8x8(const uint8_t *s, int sp, const uint8_t *r, int rp) {