Fix unsigned integer overflow in sse computation
The functions, encoder_variance and encoder_highbd_8_variance,
computed the sse value for the area beyond that of width and
height aligned to 16. As the intermediate sse value computed in
these functions was stored in an unsigned integer format, this
value could exceed the unsigned integer limit. In this CL, the
possible data overflow is avoided by changing the datatype of
the relevant variables to int64_t. The functions encoder_variance
and encoder_highbd_8_variance have been renamed to encoder_sse and
encoder_highbd_8_sse respectively.
BUG=aomedia:3369
Change-Id: I64cbb31aead6d23c1e0112400581a8a0392186ed
(cherry picked from commit 8dc64009718b23a1aa185fce5b5d8527ffb82ce5)
diff --git a/aom_dsp/psnr.c b/aom_dsp/psnr.c
index d846a10..08fb69c 100644
--- a/aom_dsp/psnr.c
+++ b/aom_dsp/psnr.c
@@ -26,59 +26,41 @@
}
}
-static void encoder_variance(const uint8_t *a, int a_stride, const uint8_t *b,
- int b_stride, int w, int h, unsigned int *sse,
- int *sum) {
+static int64_t encoder_sse(const uint8_t *a, int a_stride, const uint8_t *b,
+ int b_stride, int w, int h) {
int i, j;
-
- *sum = 0;
- *sse = 0;
+ int64_t sse = 0;
for (i = 0; i < h; i++) {
for (j = 0; j < w; j++) {
const int diff = a[j] - b[j];
- *sum += diff;
- *sse += diff * diff;
+ sse += diff * diff;
}
a += a_stride;
b += b_stride;
}
+ return sse;
}
#if CONFIG_AV1_HIGHBITDEPTH
-static void encoder_highbd_variance64(const uint8_t *a8, int a_stride,
- const uint8_t *b8, int b_stride, int w,
- int h, uint64_t *sse, int64_t *sum) {
+static int64_t encoder_highbd_8_sse(const uint8_t *a8, int a_stride,
+ const uint8_t *b8, int b_stride, int w,
+ int h) {
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
const uint16_t *b = CONVERT_TO_SHORTPTR(b8);
- int64_t tsum = 0;
- uint64_t tsse = 0;
+ int64_t sse = 0;
for (int i = 0; i < h; ++i) {
- int32_t lsum = 0;
for (int j = 0; j < w; ++j) {
const int diff = a[j] - b[j];
- lsum += diff;
- tsse += (uint32_t)(diff * diff);
+ sse += diff * diff;
}
- tsum += lsum;
a += a_stride;
b += b_stride;
}
- *sum = tsum;
- *sse = tsse;
+ return sse;
}
-static void encoder_highbd_8_variance(const uint8_t *a8, int a_stride,
- const uint8_t *b8, int b_stride, int w,
- int h, unsigned int *sse, int *sum) {
- uint64_t sse_long = 0;
- int64_t sum_long = 0;
- encoder_highbd_variance64(a8, a_stride, b8, b_stride, w, h, &sse_long,
- &sum_long);
- *sse = (unsigned int)sse_long;
- *sum = (int)sum_long;
-}
#endif // CONFIG_AV1_HIGHBITDEPTH
static int64_t get_sse(const uint8_t *a, int a_stride, const uint8_t *b,
@@ -86,26 +68,23 @@
const int dw = width % 16;
const int dh = height % 16;
int64_t total_sse = 0;
- unsigned int sse = 0;
- int sum = 0;
int x, y;
if (dw > 0) {
- encoder_variance(&a[width - dw], a_stride, &b[width - dw], b_stride, dw,
- height, &sse, &sum);
- total_sse += sse;
+ total_sse += encoder_sse(&a[width - dw], a_stride, &b[width - dw], b_stride,
+ dw, height);
}
if (dh > 0) {
- encoder_variance(&a[(height - dh) * a_stride], a_stride,
- &b[(height - dh) * b_stride], b_stride, width - dw, dh,
- &sse, &sum);
- total_sse += sse;
+ total_sse +=
+ encoder_sse(&a[(height - dh) * a_stride], a_stride,
+ &b[(height - dh) * b_stride], b_stride, width - dw, dh);
}
for (y = 0; y < height / 16; ++y) {
const uint8_t *pa = a;
const uint8_t *pb = b;
+ unsigned int sse;
for (x = 0; x < width / 16; ++x) {
aom_mse16x16(pa, a_stride, pb, b_stride, &sse);
total_sse += sse;
@@ -147,22 +126,21 @@
int x, y;
const int dw = width % 16;
const int dh = height % 16;
- unsigned int sse = 0;
- int sum = 0;
+
if (dw > 0) {
- encoder_highbd_8_variance(&a[width - dw], a_stride, &b[width - dw],
- b_stride, dw, height, &sse, &sum);
- total_sse += sse;
+ total_sse += encoder_highbd_8_sse(&a[width - dw], a_stride, &b[width - dw],
+ b_stride, dw, height);
}
if (dh > 0) {
- encoder_highbd_8_variance(&a[(height - dh) * a_stride], a_stride,
- &b[(height - dh) * b_stride], b_stride,
- width - dw, dh, &sse, &sum);
- total_sse += sse;
+ total_sse += encoder_highbd_8_sse(&a[(height - dh) * a_stride], a_stride,
+ &b[(height - dh) * b_stride], b_stride,
+ width - dw, dh);
}
+
for (y = 0; y < height / 16; ++y) {
const uint8_t *pa = a;
const uint8_t *pb = b;
+ unsigned int sse;
for (x = 0; x < width / 16; ++x) {
aom_highbd_8_mse16x16(pa, a_stride, pb, b_stride, &sse);
total_sse += sse;
