fdct4x4_test: fix unsigned overflow

The difference between src and dst will be signed, the error will be
unsigned. The change quiets -fsanitize=integer:
    unsigned integer overflow: 4294967295 * 4294967295

Change-Id: I131cefcc9583ee8a5b98eb5182fd30e9c7237ea0

test/dct16x16_test.cc

diff --git a/test/dct16x16_test.cc b/test/dct16x16_test.cc
index 9811955..b4bb14f 100644
--- a/test/dct16x16_test.cc
+++ b/test/dct16x16_test.cc
@@ -533,10 +533,10 @@
 
       for (int j = 0; j < kNumCoeffs; ++j) {
 #if CONFIG_AOM_HIGHBITDEPTH
-        const uint32_t diff =
+        const int diff =
             bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
 #else
-        const uint32_t diff = dst[j] - src[j];
+        const int diff = dst[j] - src[j];
 #endif  // CONFIG_AOM_HIGHBITDEPTH
         const uint32_t error = diff * diff;
         EXPECT_GE(1u, error) << "Error: 16x16 IDCT has error " << error
@@ -589,10 +589,10 @@
 
       for (int j = 0; j < kNumCoeffs; ++j) {
 #if CONFIG_AOM_HIGHBITDEPTH
-        const uint32_t diff =
+        const int diff =
             bit_depth_ == AOM_BITS_8 ? dst[j] - ref[j] : dst16[j] - ref16[j];
 #else
-        const uint32_t diff = dst[j] - ref[j];
+        const int diff = dst[j] - ref[j];
 #endif  // CONFIG_AOM_HIGHBITDEPTH
         const uint32_t error = diff * diff;
         EXPECT_EQ(0u, error) << "Error: 16x16 IDCT Comparison has error "

test/transform_test_base.h

diff --git a/test/transform_test_base.h b/test/transform_test_base.h
index 195058e..540136c 100644
--- a/test/transform_test_base.h
+++ b/test/transform_test_base.h
@@ -90,11 +90,11 @@
 
       for (int j = 0; j < num_coeffs_; ++j) {
 #if CONFIG_AOM_HIGHBITDEPTH
-        const uint32_t diff =
+        const int diff =
             bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
 #else
         ASSERT_EQ(AOM_BITS_8, bit_depth_);
-        const uint32_t diff = dst[j] - src[j];
+        const int diff = dst[j] - src[j];
 #endif
         const uint32_t error = diff * diff;
         if (max_error < error) max_error = error;
@@ -309,10 +309,10 @@
 
       for (int j = 0; j < num_coeffs_; ++j) {
 #if CONFIG_AOM_HIGHBITDEPTH
-        const uint32_t diff =
+        const int diff =
             bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
 #else
-        const uint32_t diff = dst[j] - src[j];
+        const int diff = dst[j] - src[j];
 #endif
         const uint32_t error = diff * diff;
         EXPECT_GE(static_cast<uint32_t>(limit), error)