Prevent negative variance
Due to rounding used computation, HDB variance computation may produce
slightly negative values. This commit adds clamping to make sure
output variance values for 10 and 12 to be non-negative.
Change-Id: Id679aa55a4c201958c4c7d28cd8733b9246a71c8
diff --git a/vpx_dsp/variance.c b/vpx_dsp/variance.c
index cdec0db..d960c54 100644
--- a/vpx_dsp/variance.c
+++ b/vpx_dsp/variance.c
@@ -341,8 +341,10 @@
int b_stride, \
uint32_t *sse) { \
int sum; \
+ int64_t var; \
highbd_10_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
} \
\
uint32_t vpx_highbd_12_variance##W##x##H##_c(const uint8_t *a, \
@@ -351,8 +353,10 @@
int b_stride, \
uint32_t *sse) { \
int sum; \
+ int64_t var; \
highbd_12_variance(a, a_stride, b, b_stride, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
}
#define HIGHBD_GET_VAR(S) \
