Fix a signed overflow in vp9 motion cost. Change-Id: I5975e3aede62202d8ee6ced33889350c0a56554a
diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c index 84ef1b4..4004dd3 100644 --- a/vp9/encoder/vp9_mcomp.c +++ b/vp9/encoder/vp9_mcomp.c
@@ -86,7 +86,9 @@ if (mvcost) { const MV diff = { mv->row - ref->row, mv->col - ref->col }; - return ROUND_POWER_OF_TWO(mv_cost(&diff, mvjcost, mvcost) * + // TODO(aconverse): See if this shift needs to be tied to + // VP9_PROB_COST_SHIFT. + return ROUND_POWER_OF_TWO((unsigned)mv_cost(&diff, mvjcost, mvcost) * error_per_bit, 13); } return 0; @@ -96,8 +98,9 @@ int error_per_bit) { const MV diff = { mv->row - ref->row, mv->col - ref->col }; - return ROUND_POWER_OF_TWO(mv_cost(&diff, x->nmvjointsadcost, - x->nmvsadcost) * error_per_bit, 8); + // TODO(aconverse): See if this shift needs to be tied to VP9_PROB_COST_SHIFT. + return ROUND_POWER_OF_TWO((unsigned)mv_cost(&diff, x->nmvjointsadcost, + x->nmvsadcost) * error_per_bit, 8); } void vp9_init_dsmotion_compensation(search_site_config *cfg, int stride) {