Add clamping in half_btf()
BUG=69073461
Change-Id: Ib28b41adfa2738681357903a81a89bcab01c87b3
(cherry picked from commit 08b26a8a257e54210d8bbdba799980bc291f368e)
diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h
index eaa0298..2e13fb1 100644
--- a/av1/common/av1_txfm.h
+++ b/av1/common/av1_txfm.h
@@ -82,7 +82,7 @@
static INLINE int32_t round_shift(int32_t value, int bit) {
assert(bit >= 1);
- return (value + (1 << (bit - 1))) >> bit;
+ return (int32_t)(((int64_t)value + (1 << (bit - 1))) >> bit);
}
static INLINE void round_shift_array(int32_t *arr, int size, int bit) {
@@ -104,7 +104,8 @@
static INLINE int32_t half_btf(int32_t w0, int32_t in0, int32_t w1, int32_t in1,
int bit) {
- int32_t result_32 = (int32_t)((int64_t)w0 * in0 + w1 * in1);
+ int32_t result_32 = (int32_t)clamp64((int64_t)w0 * in0 + (int64_t)w1 * in1,
+ INT32_MIN, INT32_MAX);
return round_shift(result_32, bit);
}