Fix overflow in clamp_mv_to_umv_border_sb
BUG=aomedia:2226
Change-Id: I4b252fbd9ab7367b0fd14d930f241cd0026e9c98
diff --git a/av1/common/reconinter.h b/av1/common/reconinter.h
index b98b0ca..5550cf2 100644
--- a/av1/common/reconinter.h
+++ b/av1/common/reconinter.h
@@ -222,9 +222,9 @@
MACROBLOCKD *xd, int can_use_previous);
// TODO(jkoleszar): yet another mv clamping function :-(
-static INLINE MV clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd,
- const MV *src_mv, int bw, int bh,
- int ss_x, int ss_y) {
+static INLINE MV32 clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd,
+ const MV *src_mv, int bw, int bh,
+ int ss_x, int ss_y) {
// If the MV points so far into the UMV border that no visible pixels
// are used for reconstruction, the subpel part of the MV can be
// discarded and the MV limited to 16 pixels with equivalent results.
@@ -232,15 +232,17 @@
const int spel_right = spel_left - SUBPEL_SHIFTS;
const int spel_top = (AOM_INTERP_EXTEND + bh) << SUBPEL_BITS;
const int spel_bottom = spel_top - SUBPEL_SHIFTS;
- MV clamped_mv = { (int16_t)(src_mv->row * (1 << (1 - ss_y))),
- (int16_t)(src_mv->col * (1 << (1 - ss_x))) };
+ const int mv_row = src_mv->row * (1 << (1 - ss_y));
+ const int mv_col = src_mv->col * (1 << (1 - ss_x));
assert(ss_x <= 1);
assert(ss_y <= 1);
- clamp_mv(&clamped_mv, xd->mb_to_left_edge * (1 << (1 - ss_x)) - spel_left,
- xd->mb_to_right_edge * (1 << (1 - ss_x)) + spel_right,
- xd->mb_to_top_edge * (1 << (1 - ss_y)) - spel_top,
- xd->mb_to_bottom_edge * (1 << (1 - ss_y)) + spel_bottom);
+ MV32 clamped_mv = {
+ clamp(mv_row, xd->mb_to_top_edge * (1 << (1 - ss_y)) - spel_top,
+ xd->mb_to_bottom_edge * (1 << (1 - ss_y)) + spel_bottom),
+ clamp(mv_col, xd->mb_to_left_edge * (1 << (1 - ss_x)) - spel_left,
+ xd->mb_to_right_edge * (1 << (1 - ss_x)) + spel_right)
+ };
return clamped_mv;
}
diff --git a/av1/common/scale.c b/av1/common/scale.c
index c525fe2..5bac53c 100644
--- a/av1/common/scale.c
+++ b/av1/common/scale.c
@@ -54,7 +54,7 @@
}
// Note: x and y are integer precision, mvq4 is q4 precision.
-MV32 av1_scale_mv(const MV *mvq4, int x, int y,
+MV32 av1_scale_mv(const MV32 *mvq4, int x, int y,
const struct scale_factors *sf) {
const int x_off_q4 = scaled_x(x << SUBPEL_BITS, sf);
const int y_off_q4 = scaled_y(y << SUBPEL_BITS, sf);
diff --git a/av1/common/scale.h b/av1/common/scale.h
index 748e958..edaad42 100644
--- a/av1/common/scale.h
+++ b/av1/common/scale.h
@@ -39,7 +39,7 @@
aom_highbd_convolve_fn_t highbd_convolve[2][2][2];
};
-MV32 av1_scale_mv(const MV *mv, int x, int y, const struct scale_factors *sf);
+MV32 av1_scale_mv(const MV32 *mv, int x, int y, const struct scale_factors *sf);
void av1_setup_scale_factors_for_frame(struct scale_factors *sf, int other_w,
int other_h, int this_w, int this_h);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index e36c9fe..e2001c0 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -591,7 +591,7 @@
block->y1 =
((pos_y + (bh - 1) * subpel_params->ys) >> SCALE_SUBPEL_BITS) + 1;
- MV temp_mv;
+ MV32 temp_mv;
temp_mv = clamp_mv_to_umv_border_sb(xd, &mv, bw, bh, pd->subsampling_x,
pd->subsampling_y);
*scaled_mv = av1_scale_mv(&temp_mv, (mi_x + x), (mi_y + y), sf);
@@ -605,7 +605,7 @@
int pos_x = (pre_x + x) << SUBPEL_BITS;
int pos_y = (pre_y + y) << SUBPEL_BITS;
- const MV mv_q4 = clamp_mv_to_umv_border_sb(
+ const MV32 mv_q4 = clamp_mv_to_umv_border_sb(
xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y);
subpel_params->xs = subpel_params->ys = SCALE_SUBPEL_SHIFTS;
subpel_params->subpel_x = (mv_q4.col & SUBPEL_MASK) << SCALE_EXTRA_BITS;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 673f4a8..3c1cd8f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -8466,7 +8466,7 @@
struct macroblockd_plane *const pd = &xd->plane[k];
const int bw = pd->width;
const int bh = pd->height;
- const MV mv_q4 = clamp_mv_to_umv_border_sb(
+ const MV32 mv_q4 = clamp_mv_to_umv_border_sb(
xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y);
const int sub_x = (mv_q4.col & SUBPEL_MASK) << SCALE_EXTRA_BITS;
const int sub_y = (mv_q4.row & SUBPEL_MASK) << SCALE_EXTRA_BITS;
diff --git a/av1/encoder/reconinter_enc.c b/av1/encoder/reconinter_enc.c
index 10ee035..00f9b24 100644
--- a/av1/encoder/reconinter_enc.c
+++ b/av1/encoder/reconinter_enc.c
@@ -62,7 +62,7 @@
subpel_params->xs = sf->x_step_q4;
subpel_params->ys = sf->y_step_q4;
} else {
- const MV mv_q4 = clamp_mv_to_umv_border_sb(
+ const MV32 mv_q4 = clamp_mv_to_umv_border_sb(
xd, &mv, bw, bh, pd->subsampling_x, pd->subsampling_y);
subpel_params->xs = subpel_params->ys = SCALE_SUBPEL_SHIFTS;
subpel_params->subpel_x = (mv_q4.col & SUBPEL_MASK) << SCALE_EXTRA_BITS;
@@ -311,8 +311,8 @@
enum mv_precision precision, int x, int y,
const MACROBLOCKD *xd, int can_use_previous) {
const int is_q4 = precision == MV_PRECISION_Q4;
- const MV mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2,
- is_q4 ? src_mv->col : src_mv->col * 2 };
+ const MV32 mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2,
+ is_q4 ? src_mv->col : src_mv->col * 2 };
MV32 mv = av1_scale_mv(&mv_q4, x, y, sf);
mv.col += SCALE_EXTRA_OFF;
mv.row += SCALE_EXTRA_OFF;
