Fix overflow and out of range integers in affine refinement
This is a fix of #507 and #479. Previously, affine refinement can possibly cause a signed integer overflow in the TMVP derivation, and an out of range issue in parameter calculation. These are fixed in this MR.
No stats change in RA tests on A2 and A4.
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c
index 25a4602..da76aec 100644
--- a/av1/common/mvref_common.c
+++ b/av1/common/mvref_common.c
@@ -179,24 +179,28 @@
const int32_t src_x = mi_col * MI_SIZE + w * 8 + 4;
const int32_t src_y = mi_row * MI_SIZE + h * 8 + 4;
#if CONFIG_AFFINE_REFINEMENT_SB
- const int32_t dst_x =
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[2] * src_x +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[3] * src_y +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[0];
- const int32_t dst_y =
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[4] * src_x +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[5] * src_y +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[1];
+ const int64_t dst_x =
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[2] * src_x +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[3] * src_y +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[0];
+ const int64_t dst_y =
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[4] * src_x +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[5] * src_y +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[1];
#else
- const int32_t dst_x = mi->wm_params[idx].wmmat[2] * src_x +
- mi->wm_params[idx].wmmat[3] * src_y +
- mi->wm_params[idx].wmmat[0];
- const int32_t dst_y = mi->wm_params[idx].wmmat[4] * src_x +
- mi->wm_params[idx].wmmat[5] * src_y +
- mi->wm_params[idx].wmmat[1];
+ const int64_t dst_x = (int64_t)mi->wm_params[idx].wmmat[2] * src_x +
+ (int64_t)mi->wm_params[idx].wmmat[3] * src_y +
+ (int64_t)mi->wm_params[idx].wmmat[0];
+ const int64_t dst_y = (int64_t)mi->wm_params[idx].wmmat[4] * src_x +
+ (int64_t)mi->wm_params[idx].wmmat[5] * src_y +
+ (int64_t)mi->wm_params[idx].wmmat[1];
#endif // CONFIG_AFFINE_REFINEMENT_SB
- const int32_t submv_x_hp = dst_x - (src_x << WARPEDMODEL_PREC_BITS);
- const int32_t submv_y_hp = dst_y - (src_y << WARPEDMODEL_PREC_BITS);
+ const int32_t submv_x_hp = (int32_t)clamp64(
+ dst_x - ((int64_t)src_x << WARPEDMODEL_PREC_BITS), INT32_MIN,
+ INT32_MAX);
+ const int32_t submv_y_hp = (int32_t)clamp64(
+ dst_y - ((int64_t)src_y << WARPEDMODEL_PREC_BITS), INT32_MIN,
+ INT32_MAX);
const int mv_offset_y =
ROUND_POWER_OF_TWO_SIGNED(submv_y_hp, WARPEDMODEL_PREC_BITS - 3);
const int mv_offset_x =
@@ -424,24 +428,26 @@
const int32_t src_x = mi_col * MI_SIZE + w * 8 + 4;
const int32_t src_y = mi_row * MI_SIZE + h * 8 + 4;
#if CONFIG_AFFINE_REFINEMENT_SB
- const int32_t dst_x =
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[2] * src_x +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[3] * src_y +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[0];
- const int32_t dst_y =
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[4] * src_x +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[5] * src_y +
- xd->wm_params_sb[2 * sb_idx + idx].wmmat[1];
+ const int64_t dst_x =
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[2] * src_x +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[3] * src_y +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[0];
+ const int64_t dst_y =
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[4] * src_x +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[5] * src_y +
+ (int64_t)xd->wm_params_sb[2 * sb_idx + idx].wmmat[1];
#else
- const int32_t dst_x = mi->wm_params[idx].wmmat[2] * src_x +
- mi->wm_params[idx].wmmat[3] * src_y +
- mi->wm_params[idx].wmmat[0];
- const int32_t dst_y = mi->wm_params[idx].wmmat[4] * src_x +
- mi->wm_params[idx].wmmat[5] * src_y +
- mi->wm_params[idx].wmmat[1];
+ const int64_t dst_x = (int64_t)mi->wm_params[idx].wmmat[2] * src_x +
+ (int64_t)mi->wm_params[idx].wmmat[3] * src_y +
+ (int64_t)mi->wm_params[idx].wmmat[0];
+ const int64_t dst_y = (int64_t)mi->wm_params[idx].wmmat[4] * src_x +
+ (int64_t)mi->wm_params[idx].wmmat[5] * src_y +
+ (int64_t)mi->wm_params[idx].wmmat[1];
#endif // CONFIG_AFFINE_REFINEMENT_SB
- const int32_t submv_x_hp = dst_x - (src_x << WARPEDMODEL_PREC_BITS);
- const int32_t submv_y_hp = dst_y - (src_y << WARPEDMODEL_PREC_BITS);
+ const int32_t submv_x_hp = (int32_t)clamp64(
+ dst_x - (src_x << WARPEDMODEL_PREC_BITS), INT32_MIN, INT32_MAX);
+ const int32_t submv_y_hp = (int32_t)clamp64(
+ dst_y - (src_y << WARPEDMODEL_PREC_BITS), INT32_MIN, INT32_MAX);
const int mv_offset_y = ROUND_POWER_OF_TWO_SIGNED(
submv_y_hp, WARPEDMODEL_PREC_BITS - 3);
const int mv_offset_x = ROUND_POWER_OF_TWO_SIGNED(
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c
index 5634c3e..26a4c49 100644
--- a/av1/common/reconinter.c
+++ b/av1/common/reconinter.c
@@ -1847,32 +1847,46 @@
const int angle = -d * am_params->rot_angle;
cos_angle = unit_offset;
sin_angle = angle * (1 << (WARPEDMODEL_PREC_BITS - AFFINE_PREC_BITS));
- wm->wmmat[2] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(scale_x * cos_angle,
- WARPEDMODEL_PREC_BITS);
- wm->wmmat[5] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(scale_y * cos_angle,
- WARPEDMODEL_PREC_BITS);
+ wm->wmmat[2] = (int32_t)clamp64(
+ ROUND_POWER_OF_TWO_SIGNED_64(scale_x * cos_angle, WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
+ wm->wmmat[5] = (int32_t)clamp64(
+ ROUND_POWER_OF_TWO_SIGNED_64(scale_y * cos_angle, WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
if (d > 0) {
// Parameters of A^-1
- wm->wmmat[3] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(-scale_x * sin_angle,
- WARPEDMODEL_PREC_BITS);
- wm->wmmat[4] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(scale_y * sin_angle,
- WARPEDMODEL_PREC_BITS);
+ wm->wmmat[3] =
+ (int32_t)clamp64(ROUND_POWER_OF_TWO_SIGNED_64(-scale_x * sin_angle,
+ WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
+ wm->wmmat[4] =
+ (int32_t)clamp64(ROUND_POWER_OF_TWO_SIGNED_64(scale_y * sin_angle,
+ WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
int64_t tmp_tx = (int64_t)wm->wmmat[2] * (int64_t)am_params->tran_x -
(int64_t)wm->wmmat[3] * (int64_t)am_params->tran_y;
int64_t tmp_ty = (int64_t)wm->wmmat[4] * (int64_t)am_params->tran_x +
(int64_t)wm->wmmat[5] * (int64_t)am_params->tran_y;
- wm->wmmat[0] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(-d * tmp_tx,
- WARPEDMODEL_PREC_BITS);
- wm->wmmat[1] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(-d * tmp_ty,
- WARPEDMODEL_PREC_BITS);
+ wm->wmmat[0] = (int32_t)clamp64(
+ ROUND_POWER_OF_TWO_SIGNED_64(tmp_tx * (-d), WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
+ wm->wmmat[1] = (int32_t)clamp64(
+ ROUND_POWER_OF_TWO_SIGNED_64(tmp_ty * (-d), WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
} else {
// Parameters of A
- wm->wmmat[3] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(-scale_y * sin_angle,
- WARPEDMODEL_PREC_BITS);
- wm->wmmat[4] = (int32_t)ROUND_POWER_OF_TWO_SIGNED_64(scale_x * sin_angle,
- WARPEDMODEL_PREC_BITS);
- wm->wmmat[0] = -d * am_params->tran_x;
- wm->wmmat[1] = -d * am_params->tran_y;
+ wm->wmmat[3] =
+ (int32_t)clamp64(ROUND_POWER_OF_TWO_SIGNED_64(-scale_y * sin_angle,
+ WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
+ wm->wmmat[4] =
+ (int32_t)clamp64(ROUND_POWER_OF_TWO_SIGNED_64(scale_x * sin_angle,
+ WARPEDMODEL_PREC_BITS),
+ INT32_MIN, INT32_MAX);
+ wm->wmmat[0] = (int32_t)clamp64((int64_t)am_params->tran_x * (-d),
+ INT32_MIN, INT32_MAX);
+ wm->wmmat[1] = (int32_t)clamp64((int64_t)am_params->tran_y * (-d),
+ INT32_MIN, INT32_MAX);
}
wm->wmmat[0] = clamp(wm->wmmat[0], -WARPEDMODEL_TRANS_CLAMP,
WARPEDMODEL_TRANS_CLAMP - unit_offset);
