Resolve -Wshorten-64-to-32 in highbd obmc/ext-inter variance.
For 8-bit the subtrahend is small enough to fit into uint32_t.
Prevent negative variance in 10/12-bit.
Previously:
51e0c54ed Resolve -Wshorten-64-to-32 in highbd variance.
BUG=aomedia:445
Change-Id: Ibb9bfc5e926de354bbab80127012e5e1861348a3
diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index 55fac22..9fc0db7 100644
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -842,25 +842,29 @@
int sum; \
highbd_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, sse, \
&sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
unsigned int aom_highbd_10_masked_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \
const uint8_t *m, int m_stride, unsigned int *sse) { \
int sum; \
+ int64_t var; \
highbd_10_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, \
sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
} \
\
unsigned int aom_highbd_12_masked_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \
const uint8_t *m, int m_stride, unsigned int *sse) { \
int sum; \
+ int64_t var; \
highbd_12_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, \
sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
}
#define HIGHBD_MASK_SUBPIX_VAR(W, H) \
@@ -1123,23 +1127,27 @@
const int32_t *mask, unsigned int *sse) { \
int sum; \
highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
unsigned int aom_highbd_10_obmc_variance##W##x##H##_c( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
+ int64_t var; \
highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
} \
\
unsigned int aom_highbd_12_obmc_variance##W##x##H##_c( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
+ int64_t var; \
highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
}
#define HIGHBD_OBMC_SUBPIX_VAR(W, H) \
diff --git a/aom_dsp/x86/obmc_variance_sse4.c b/aom_dsp/x86/obmc_variance_sse4.c
index f7cb46d..efb3659 100644
--- a/aom_dsp/x86/obmc_variance_sse4.c
+++ b/aom_dsp/x86/obmc_variance_sse4.c
@@ -311,23 +311,27 @@
const int32_t *mask, unsigned int *sse) { \
int sum; \
highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
unsigned int aom_highbd_10_obmc_variance##W##x##H##_sse4_1( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
+ int64_t var; \
highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
} \
\
unsigned int aom_highbd_12_obmc_variance##W##x##H##_sse4_1( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
+ int64_t var; \
highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
- return *sse - (((int64_t)sum * sum) / (W * H)); \
+ var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
+ return (var >= 0) ? (uint32_t)var : 0; \
}
#if CONFIG_EXT_PARTITION
