commit 8e739bcdf4276deb8177464b62aee35bd7aa4325
author Hui Su <huisu@google.com> Thu Nov 09 14:37:50 2017 -0800
committer Hui Su <huisu@google.com> Thu Nov 23 04:41:43 2017 +0000
tree 815db9b3590f3bcb3b6a5a62697db21d9e359fb8
parent 86b75c8ab791dc11d6c79d7df3f3d112d8bdaf9c

Add clamping in half_btf()

BUG=69073461

Change-Id: Ib28b41adfa2738681357903a81a89bcab01c87b3
(cherry picked from commit 08b26a8a257e54210d8bbdba799980bc291f368e)

av1/common/av1_txfm.h

diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h
index eaa0298..2e13fb1 100644
--- a/av1/common/av1_txfm.h
+++ b/av1/common/av1_txfm.h
@@ -82,7 +82,7 @@
 
 static INLINE int32_t round_shift(int32_t value, int bit) {
   assert(bit >= 1);
-  return (value + (1 << (bit - 1))) >> bit;
+  return (int32_t)(((int64_t)value + (1 << (bit - 1))) >> bit);
 }
 
 static INLINE void round_shift_array(int32_t *arr, int size, int bit) {
@@ -104,7 +104,8 @@
 
 static INLINE int32_t half_btf(int32_t w0, int32_t in0, int32_t w1, int32_t in1,
                                int bit) {
-  int32_t result_32 = (int32_t)((int64_t)w0 * in0 + w1 * in1);
+  int32_t result_32 = (int32_t)clamp64((int64_t)w0 * in0 + (int64_t)w1 * in1,
+                                       INT32_MIN, INT32_MAX);
   return round_shift(result_32, bit);
 }