Fix integer overflows in av1_iidentity*_c() BUG=b/69238080,b/69288165 Change-Id: Ia761d4b77049a55bd8040b5ed76063b2fac750ee
diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c index 3399b7c..81253ab 100644 --- a/av1/common/av1_inv_txfm1d.c +++ b/av1/common/av1_inv_txfm1d.c
@@ -1552,14 +1552,14 @@ const int8_t *cos_bit, const int8_t *stage_range) { (void)cos_bit; for (int i = 0; i < 4; ++i) - output[i] = (int32_t)dct_const_round_shift(input[i] * Sqrt2); + output[i] = (int32_t)dct_const_round_shift(Sqrt2 * (int64_t)input[i]); range_check(0, input, output, 4, stage_range[0]); } void av1_iidentity8_c(const int32_t *input, int32_t *output, const int8_t *cos_bit, const int8_t *stage_range) { (void)cos_bit; - for (int i = 0; i < 8; ++i) output[i] = input[i] * 2; + for (int i = 0; i < 8; ++i) output[i] = (int32_t)((int64_t)input[i] * 2); range_check(0, input, output, 8, stage_range[0]); } @@ -1567,14 +1567,14 @@ const int8_t *cos_bit, const int8_t *stage_range) { (void)cos_bit; for (int i = 0; i < 16; ++i) - output[i] = (int32_t)dct_const_round_shift(input[i] * 2 * Sqrt2); + output[i] = (int32_t)dct_const_round_shift(Sqrt2 * (int64_t)input[i] * 2); range_check(0, input, output, 16, stage_range[0]); } void av1_iidentity32_c(const int32_t *input, int32_t *output, const int8_t *cos_bit, const int8_t *stage_range) { (void)cos_bit; - for (int i = 0; i < 32; ++i) output[i] = input[i] * 4; + for (int i = 0; i < 32; ++i) output[i] = (int32_t)((int64_t)input[i] * 4); range_check(0, input, output, 32, stage_range[0]); } #endif // CONFIG_EXT_TX