commit 0123bc7d94881cf3385e115394a60a9d457caec0
author David Barker <david.barker@argondesign.com> Wed Sep 06 15:22:59 2017 +0100
committer Steinar Midtskogen <stemidts@cisco.com> Mon Sep 11 07:28:10 2017 +0000
tree a5709006b95821bcb95de16d29705a24249e36ed
parent 87b01b5adddb91ad79e8b8badea891112abbb932

cdef-singlepass: Fix integer overflow

When cdef-singlepass is enabled, it is possible to signal the
CDEF parameters in such a way that we end up with unsigned
integer overflow in constrain().

Fix this by i) using signed instead of unsigned values,
and ii) clamping the result to avoid going on to shift
by a negative amount.

Change-Id: Ib677b2d644e44000c54959f7280e646bf02054da

av1/common/cdef.h

diff --git a/av1/common/cdef.h b/av1/common/cdef.h
index 50fae11..9de24bf 100644
--- a/av1/common/cdef.h
+++ b/av1/common/cdef.h
@@ -24,14 +24,12 @@
 
 static INLINE int sign(int i) { return i < 0 ? -1 : 1; }
 
-static INLINE int constrain(int diff, int threshold, unsigned int damping) {
-  return threshold
-             ? sign(diff) *
-                   AOMMIN(abs(diff),
-                          AOMMAX(0,
-                                 threshold - (abs(diff) >>
-                                              (damping - get_msb(threshold)))))
-             : 0;
+static INLINE int constrain(int diff, int threshold, int damping) {
+  if (!threshold) return 0;
+
+  const int shift = AOMMAX(0, damping - get_msb(threshold));
+  return sign(diff) *
+         AOMMIN(abs(diff), AOMMAX(0, threshold - (abs(diff) >> shift)));
 }
 
 #ifdef __cplusplus

av1/common/cdef_block_simd.h

diff --git a/av1/common/cdef_block_simd.h b/av1/common/cdef_block_simd.h
index b53b88b..aa7d3c3 100644
--- a/av1/common/cdef_block_simd.h
+++ b/av1/common/cdef_block_simd.h
@@ -268,8 +268,10 @@
   const int *pri_taps = cdef_pri_taps[pri_strength & 1];
   const int *sec_taps = cdef_sec_taps[pri_strength & 1];
 
-  if (pri_strength) pri_damping -= get_msb(pri_strength);
-  if (sec_strength) sec_damping -= get_msb(sec_strength);
+  if (pri_strength)
+    pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
+  if (sec_strength)
+    sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
 
   sum = v256_zero();
   row = v256_from_v64(v64_load_aligned(&in[0 * CDEF_BSTRIDE]),
@@ -504,8 +506,10 @@
   const int *pri_taps = cdef_pri_taps[pri_strength & 1];
   const int *sec_taps = cdef_sec_taps[pri_strength & 1];
 
-  if (pri_strength) pri_damping -= get_msb(pri_strength);
-  if (sec_strength) sec_damping -= get_msb(sec_strength);
+  if (pri_strength)
+    pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
+  if (sec_strength)
+    sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
   for (i = 0; i < 8; i += 2) {
     sum = v256_zero();
     row = v256_from_v128(v128_load_aligned(&in[i * CDEF_BSTRIDE]),
@@ -717,8 +721,10 @@
   const int *pri_taps = cdef_pri_taps[pri_strength & 1];
   const int *sec_taps = cdef_sec_taps[pri_strength & 1];
 
-  if (pri_strength) pri_damping -= get_msb(pri_strength);
-  if (sec_strength) sec_damping -= get_msb(sec_strength);
+  if (pri_strength)
+    pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
+  if (sec_strength)
+    sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
   for (i = 0; i < 4; i += 2) {
     sum = v128_zero();
     row = v128_from_v64(v64_load_aligned(&in[i * CDEF_BSTRIDE]),
@@ -888,8 +894,10 @@
   const int *pri_taps = cdef_pri_taps[pri_strength & 1];
   const int *sec_taps = cdef_sec_taps[pri_strength & 1];
 
-  if (pri_strength) pri_damping -= get_msb(pri_strength);
-  if (sec_strength) sec_damping -= get_msb(sec_strength);
+  if (pri_strength)
+    pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
+  if (sec_strength)
+    sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
 
   for (i = 0; i < 8; i++) {
     sum = v128_zero();