commit cb57c95a6a9ab5a87b0cd5c5eb3127845c046a30
author Sebastien Alaiwan <sebastien.alaiwan@allegrodvt.com> Tue Feb 28 10:50:35 2017 +0100
committer sebastien alaiwan <sebastien.alaiwan@allegrodvt.com> Wed Mar 01 08:14:21 2017 +0000
tree 93a602457c779954f25df5457b404b700f6ff2c6
parent 07e33f4ee4999158ffc8742aef32aae3cf6c6e90

Fix potential integer overflow in range checking

Change-Id: I20b951bb67f4c2147055751a8cd9534ceddc7215

av1/common/av1_inv_txfm1d.c

diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c
index b0c113e..af4ddb4 100644
--- a/av1/common/av1_inv_txfm1d.c
+++ b/av1/common/av1_inv_txfm1d.c
@@ -15,20 +15,23 @@
 
 void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
                       int32_t size, int8_t bit) {
-  const int32_t maxValue = (1 << (bit - 1)) - 1;
-  const int32_t minValue = -(1 << (bit - 1));
+  const int64_t maxValue = (1LL << (bit - 1)) - 1;
+  const int64_t minValue = -(1LL << (bit - 1));
 
   for (int i = 0; i < size; ++i) {
     if (buf[i] < minValue || buf[i] > maxValue) {
-      int buf_bit = get_max_bit(abs(buf[i])) + 1;
-      printf("======== %s %d overflow ========\n", __FILE__, __LINE__);
-      printf("stage: %d node: %d\n", stage, i);
-      printf("bit: %d buf_bit: %d buf[i]: %d\n", bit, buf_bit, buf[i]);
-      printf("input:\n");
+      fprintf(stderr, "Error: coeffs contain out-of-range values\n");
+      fprintf(stderr, "stage: %d\n", stage);
+      fprintf(stderr, "node: %d\n", i);
+      fprintf(stderr, "allowed range: [%d;%d]\n", minValue, maxValue);
+      fprintf(stderr, "coeffs: ");
+
+      fprintf(stderr, "[");
       for (int j = 0; j < size; j++) {
-        printf("%d,", input[j]);
+        if (j > 0) fprintf(stderr, ", ");
+        fprintf(stderr, "%d", input[j]);
       }
-      printf("\n");
+      fprintf(stderr, "]\n");
       assert(0);
     }
   }