Add clamping in half_btf() BUG=69073461 Change-Id: Ib28b41adfa2738681357903a81a89bcab01c87b3
diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h index 7845dee..0530b3c 100644 --- a/av1/common/av1_txfm.h +++ b/av1/common/av1_txfm.h
@@ -81,7 +81,7 @@ static INLINE int32_t round_shift(int32_t value, int bit) { assert(bit >= 1); - return (value + (1 << (bit - 1))) >> bit; + return (int32_t)(((int64_t)value + (1 << (bit - 1))) >> bit); } static INLINE void round_shift_array(int32_t *arr, int size, int bit) { @@ -103,7 +103,8 @@ static INLINE int32_t half_btf(int32_t w0, int32_t in0, int32_t w1, int32_t in1, int bit) { - int32_t result_32 = (int32_t)((int64_t)w0 * in0 + w1 * in1); + int32_t result_32 = (int32_t)clamp64((int64_t)w0 * in0 + (int64_t)w1 * in1, + INT32_MIN, INT32_MAX); #if CONFIG_COEFFICIENT_RANGE_CHECKING int64_t result_64 = (int64_t)w0 * (int64_t)in0 + (int64_t)w1 * (int64_t)in1; if (result_64 < INT32_MIN || result_64 > INT32_MAX) {