commit 3dd1c5ac45cdb3f1f600ef6c1c40302fb566fa47
author James Zern <jzern@google.com> Thu Apr 21 13:56:16 2022 -0700
committer James Zern <jzern@google.com> Fri Apr 29 20:59:47 2022 +0000
tree 8b7d0baff6699de9af39723f48deed5e6bd4aba9
parent 5c0ded6b5668db1962dc649becf545a110afccd3

pickrst,update_[ab]_sep_sym: clamp intermediates

this avoids integer 32 and 64-bit overflows due to large values returned
from linsolve_wiener().

av1/encoder/pickrst.c:1232:35: runtime error: signed integer overflow: 2
* -1901130238 cannot be represented in type 'int'

av1/encoder/pickrst.c:1150:39: runtime error: signed integer overflow:
-25909513532 * -1901130238 cannot be represented in type 'long'

no change in BDRATE in lowres, lowres2, midres, midres2 or hdres sets

Bug: b/229500034
Bug: b/229955363
Change-Id: I80db57c0ccaa05f0a1966d0de48017ac91fd4693

av1/encoder/pickrst.c

diff --git a/av1/encoder/pickrst.c b/av1/encoder/pickrst.c
index 246b0aa..d01c11c 100644
--- a/av1/encoder/pickrst.c
+++ b/av1/encoder/pickrst.c
@@ -1081,7 +1081,7 @@
 // Solve linear equations to find Wiener filter tap values
 // Taps are output scaled by WIENER_FILT_STEP
 static int linsolve_wiener(int n, int64_t *A, int stride, int64_t *b,
-                           int32_t *x) {
+                           int64_t *x) {
   for (int k = 0; k < n - 1; k++) {
     // Partial pivoting: bring the row with the largest pivot to the top
     for (int i = n - 1; i > k; i--) {
@@ -1116,7 +1116,7 @@
       c += A[i * stride + j] * x[j] / WIENER_TAP_SCALE_FACTOR;
     }
     // Store filter taps x in scaled form.
-    x[i] = (int32_t)(WIENER_TAP_SCALE_FACTOR * (b[i] - c) / A[i * stride + i]);
+    x[i] = WIENER_TAP_SCALE_FACTOR * (b[i] - c) / A[i * stride + i];
   }
 
   return 1;
@@ -1126,7 +1126,7 @@
 static AOM_INLINE void update_a_sep_sym(int wiener_win, int64_t **Mc,
                                         int64_t **Hc, int32_t *a, int32_t *b) {
   int i, j;
-  int32_t S[WIENER_WIN];
+  int64_t S[WIENER_WIN];
   int64_t A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1];
   const int wiener_win2 = wiener_win * wiener_win;
   const int wiener_halfwin1 = (wiener_win >> 1) + 1;
@@ -1174,7 +1174,10 @@
       S[i] = S[wiener_win - 1 - i];
       S[wiener_halfwin1 - 1] -= 2 * S[i];
     }
-    memcpy(a, S, wiener_win * sizeof(*a));
+    for (i = 0; i < wiener_win; ++i) {
+      a[i] = (int32_t)CLIP(S[i], -(1 << (WIENER_FILT_BITS - 1)),
+                           (1 << (WIENER_FILT_BITS - 1)) - 1);
+    }
   }
 }
 
@@ -1182,7 +1185,7 @@
 static AOM_INLINE void update_b_sep_sym(int wiener_win, int64_t **Mc,
                                         int64_t **Hc, int32_t *a, int32_t *b) {
   int i, j;
-  int32_t S[WIENER_WIN];
+  int64_t S[WIENER_WIN];
   int64_t A[WIENER_HALFWIN1], B[WIENER_HALFWIN1 * WIENER_HALFWIN1];
   const int wiener_win2 = wiener_win * wiener_win;
   const int wiener_halfwin1 = (wiener_win >> 1) + 1;
@@ -1231,7 +1234,10 @@
       S[i] = S[wiener_win - 1 - i];
       S[wiener_halfwin1 - 1] -= 2 * S[i];
     }
-    memcpy(b, S, wiener_win * sizeof(*b));
+    for (i = 0; i < wiener_win; ++i) {
+      b[i] = (int32_t)CLIP(S[i], -(1 << (WIENER_FILT_BITS - 1)),
+                           (1 << (WIENER_FILT_BITS - 1)) - 1);
+    }
   }
 }