Avoid an unsigned overflow in invert_quant Change-Id: I16a570b2af66b6580d1cd6f8345a25f079009bf4
diff --git a/vp10/encoder/quantize.c b/vp10/encoder/quantize.c index 86b324f..136efe3 100644 --- a/vp10/encoder/quantize.c +++ b/vp10/encoder/quantize.c
@@ -219,12 +219,12 @@ static void invert_quant(int16_t *quant, int16_t *shift, int d) { unsigned t; - int l; + int l, m; t = d; for (l = 0; t > 1; l++) t >>= 1; - t = 1 + (1 << (16 + l)) / d; - *quant = (int16_t)(t - (1 << 16)); + m = 1 + (1 << (16 + l)) / d; + *quant = (int16_t)(m - (1 << 16)); *shift = 1 << (16 - l); }
diff --git a/vp8/encoder/vp8_quantize.c b/vp8/encoder/vp8_quantize.c index ee922c9..0d101ba 100644 --- a/vp8/encoder/vp8_quantize.c +++ b/vp8/encoder/vp8_quantize.c
@@ -227,12 +227,12 @@ if(improved_quant) { unsigned t; - int l; + int l, m; t = d; for(l = 0; t > 1; l++) t>>=1; - t = 1 + (1<<(16+l))/d; - *quant = (short)(t - (1<<16)); + m = 1 + (1<<(16+l))/d; + *quant = (short)(m - (1<<16)); *shift = l; /* use multiplication and constant shift by 16 */ *shift = 1 << (16 - *shift);
diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c index 91f877e..9766c05 100644 --- a/vp9/encoder/vp9_quantize.c +++ b/vp9/encoder/vp9_quantize.c
@@ -219,12 +219,12 @@ static void invert_quant(int16_t *quant, int16_t *shift, int d) { unsigned t; - int l; + int l, m; t = d; for (l = 0; t > 1; l++) t >>= 1; - t = 1 + (1 << (16 + l)) / d; - *quant = (int16_t)(t - (1 << 16)); + m = 1 + (1 << (16 + l)) / d; + *quant = (int16_t)(m - (1 << 16)); *shift = 1 << (16 - l); }