fdct4x4_test: fix unsigned overflow
The difference between src and dst will be signed, the error will be
unsigned. The change quiets -fsanitize=integer:
unsigned integer overflow: 4294967295 * 4294967295
Change-Id: Ide662f26fae682309a3474243f8a5b5ccea2049e
diff --git a/test/fdct4x4_test.cc b/test/fdct4x4_test.cc
index 3915d29..7c7aa8a 100644
--- a/test/fdct4x4_test.cc
+++ b/test/fdct4x4_test.cc
@@ -253,10 +253,10 @@
for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
- const uint32_t diff =
+ const int diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else
- const uint32_t diff = dst[j] - src[j];
+ const int diff = dst[j] - src[j];
#endif
const uint32_t error = diff * diff;
EXPECT_GE(static_cast<uint32_t>(limit), error)