Fix overflow in clamp_mv_to_umv_border_sb BUG=aomedia:2226 Change-Id: I4b252fbd9ab7367b0fd14d930f241cd0026e9c98
diff --git a/av1/common/reconinter.h b/av1/common/reconinter.h index b98b0ca..5550cf2 100644 --- a/av1/common/reconinter.h +++ b/av1/common/reconinter.h
@@ -222,9 +222,9 @@ MACROBLOCKD *xd, int can_use_previous); // TODO(jkoleszar): yet another mv clamping function :-( -static INLINE MV clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd, - const MV *src_mv, int bw, int bh, - int ss_x, int ss_y) { +static INLINE MV32 clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd, + const MV *src_mv, int bw, int bh, + int ss_x, int ss_y) { // If the MV points so far into the UMV border that no visible pixels // are used for reconstruction, the subpel part of the MV can be // discarded and the MV limited to 16 pixels with equivalent results. @@ -232,15 +232,17 @@ const int spel_right = spel_left - SUBPEL_SHIFTS; const int spel_top = (AOM_INTERP_EXTEND + bh) << SUBPEL_BITS; const int spel_bottom = spel_top - SUBPEL_SHIFTS; - MV clamped_mv = { (int16_t)(src_mv->row * (1 << (1 - ss_y))), - (int16_t)(src_mv->col * (1 << (1 - ss_x))) }; + const int mv_row = src_mv->row * (1 << (1 - ss_y)); + const int mv_col = src_mv->col * (1 << (1 - ss_x)); assert(ss_x <= 1); assert(ss_y <= 1); - clamp_mv(&clamped_mv, xd->mb_to_left_edge * (1 << (1 - ss_x)) - spel_left, - xd->mb_to_right_edge * (1 << (1 - ss_x)) + spel_right, - xd->mb_to_top_edge * (1 << (1 - ss_y)) - spel_top, - xd->mb_to_bottom_edge * (1 << (1 - ss_y)) + spel_bottom); + MV32 clamped_mv = { + clamp(mv_row, xd->mb_to_top_edge * (1 << (1 - ss_y)) - spel_top, + xd->mb_to_bottom_edge * (1 << (1 - ss_y)) + spel_bottom), + clamp(mv_col, xd->mb_to_left_edge * (1 << (1 - ss_x)) - spel_left, + xd->mb_to_right_edge * (1 << (1 - ss_x)) + spel_right) + }; return clamped_mv; }
diff --git a/av1/common/scale.c b/av1/common/scale.c index c525fe2..5bac53c 100644 --- a/av1/common/scale.c +++ b/av1/common/scale.c
@@ -54,7 +54,7 @@ } // Note: x and y are integer precision, mvq4 is q4 precision. -MV32 av1_scale_mv(const MV *mvq4, int x, int y, +MV32 av1_scale_mv(const MV32 *mvq4, int x, int y, const struct scale_factors *sf) { const int x_off_q4 = scaled_x(x << SUBPEL_BITS, sf); const int y_off_q4 = scaled_y(y << SUBPEL_BITS, sf);
diff --git a/av1/common/scale.h b/av1/common/scale.h index 748e958..edaad42 100644 --- a/av1/common/scale.h +++ b/av1/common/scale.h
@@ -39,7 +39,7 @@ aom_highbd_convolve_fn_t highbd_convolve[2][2][2]; }; -MV32 av1_scale_mv(const MV *mv, int x, int y, const struct scale_factors *sf); +MV32 av1_scale_mv(const MV32 *mv, int x, int y, const struct scale_factors *sf); void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w, int other_h, int this_w, int this_h);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index e36c9fe..e2001c0 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -591,7 +591,7 @@ block->y1 = ((pos_y + (bh - 1) * subpel_params->ys) >> SCALE_SUBPEL_BITS) + 1; - MV temp_mv; + MV32 temp_mv; temp_mv = clamp_mv_to_umv_border_sb(xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y); *scaled_mv = av1_scale_mv(&temp_mv, (mi_x + x), (mi_y + y), sf); @@ -605,7 +605,7 @@ int pos_x = (pre_x + x) << SUBPEL_BITS; int pos_y = (pre_y + y) << SUBPEL_BITS; - const MV mv_q4 = clamp_mv_to_umv_border_sb( + const MV32 mv_q4 = clamp_mv_to_umv_border_sb( xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y); subpel_params->xs = subpel_params->ys = SCALE_SUBPEL_SHIFTS; subpel_params->subpel_x = (mv_q4.col & SUBPEL_MASK) << SCALE_EXTRA_BITS;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 673f4a8..3c1cd8f 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -8466,7 +8466,7 @@ struct macroblockd_plane *const pd = &xd->plane[k]; const int bw = pd->width; const int bh = pd->height; - const MV mv_q4 = clamp_mv_to_umv_border_sb( + const MV32 mv_q4 = clamp_mv_to_umv_border_sb( xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y); const int sub_x = (mv_q4.col & SUBPEL_MASK) << SCALE_EXTRA_BITS; const int sub_y = (mv_q4.row & SUBPEL_MASK) << SCALE_EXTRA_BITS;
diff --git a/av1/encoder/reconinter_enc.c b/av1/encoder/reconinter_enc.c index 10ee035..00f9b24 100644 --- a/av1/encoder/reconinter_enc.c +++ b/av1/encoder/reconinter_enc.c
@@ -62,7 +62,7 @@ subpel_params->xs = sf->x_step_q4; subpel_params->ys = sf->y_step_q4; } else { - const MV mv_q4 = clamp_mv_to_umv_border_sb( + const MV32 mv_q4 = clamp_mv_to_umv_border_sb( xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y); subpel_params->xs = subpel_params->ys = SCALE_SUBPEL_SHIFTS; subpel_params->subpel_x = (mv_q4.col & SUBPEL_MASK) << SCALE_EXTRA_BITS; @@ -311,8 +311,8 @@ enum mv_precision precision, int x, int y, const MACROBLOCKD *xd, int can_use_previous) { const int is_q4 = precision == MV_PRECISION_Q4; - const MV mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, - is_q4 ? src_mv->col : src_mv->col * 2 }; + const MV32 mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, + is_q4 ? src_mv->col : src_mv->col * 2 }; MV32 mv = av1_scale_mv(&mv_q4, x, y, sf); mv.col += SCALE_EXTRA_OFF; mv.row += SCALE_EXTRA_OFF;