Fix MV out of range bug
While encoding ultra high resolution(8k) video, the MV can go out of
range. The MVs obtained from single refernece frame modes are used
in compound reference modes. We need to check if the single reference MV
is valid, and also need to clamp it if it is out of range. Otherwise,
out of range MV is created.
This patch fixed the above bug.
BUG=aomedia:2408
This would potentially cause STATS_CHANGED.
Change-Id: Ibbaa33dc850b86563ab8e5448e4dec97e73e8371
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index d7a4040..00ce953 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -59,21 +59,6 @@
if (mv_limits->row_max > row_max) mv_limits->row_max = row_max;
}
-static void set_subpel_mv_search_range(const MvLimits *mv_limits, int *col_min,
- int *col_max, int *row_min, int *row_max,
- const MV *ref_mv) {
- const int max_mv = MAX_FULL_PEL_VAL * 8;
- const int minc = AOMMAX(mv_limits->col_min * 8, ref_mv->col - max_mv);
- const int maxc = AOMMIN(mv_limits->col_max * 8, ref_mv->col + max_mv);
- const int minr = AOMMAX(mv_limits->row_min * 8, ref_mv->row - max_mv);
- const int maxr = AOMMIN(mv_limits->row_max * 8, ref_mv->row + max_mv);
-
- *col_min = AOMMAX(MV_LOW + 1, minc);
- *col_max = AOMMIN(MV_UPP - 1, maxc);
- *row_min = AOMMAX(MV_LOW + 1, minr);
- *row_max = AOMMIN(MV_UPP - 1, maxr);
-}
-
int av1_init_search_range(int size) {
int sr = 0;
// Minimum search size no matter what the passed in value.
diff --git a/av1/encoder/mcomp.h b/av1/encoder/mcomp.h
index de4d931..1b28426 100644
--- a/av1/encoder/mcomp.h
+++ b/av1/encoder/mcomp.h
@@ -182,6 +182,22 @@
}
}
+static INLINE void set_subpel_mv_search_range(const MvLimits *mv_limits,
+ int *col_min, int *col_max,
+ int *row_min, int *row_max,
+ const MV *ref_mv) {
+ const int max_mv = MAX_FULL_PEL_VAL * 8;
+ const int minc = AOMMAX(mv_limits->col_min * 8, ref_mv->col - max_mv);
+ const int maxc = AOMMIN(mv_limits->col_max * 8, ref_mv->col + max_mv);
+ const int minr = AOMMAX(mv_limits->row_min * 8, ref_mv->row - max_mv);
+ const int maxr = AOMMIN(mv_limits->row_max * 8, ref_mv->row + max_mv);
+
+ *col_min = AOMMAX(MV_LOW + 1, minc);
+ *col_max = AOMMIN(MV_UPP - 1, maxc);
+ *row_min = AOMMAX(MV_LOW + 1, minr);
+ *row_max = AOMMIN(MV_UPP - 1, maxr);
+}
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 8c98cd5..537fee9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -7984,6 +7984,18 @@
return !mv_check_bounds(&x->mv_limits, &out_mv->as_mv);
}
+// To use single newmv directly for compound modes, need to clamp the mv to the
+// valid mv range. Without this, encoder would generate out of range mv, and
+// this is seen in 8k encoding.
+static INLINE void clamp_mv_in_range(MACROBLOCK *const x, int_mv *mv,
+ int ref_idx) {
+ const int_mv ref_mv = av1_get_ref_mv(x, ref_idx);
+ int minc, maxc, minr, maxr;
+ set_subpel_mv_search_range(&x->mv_limits, &minc, &maxc, &minr, &maxr,
+ &ref_mv.as_mv);
+ clamp_mv(&mv->as_mv, minc, maxc, minr, maxr);
+}
+
static int64_t handle_newmv(const AV1_COMP *const cpi, MACROBLOCK *const x,
const BLOCK_SIZE bsize, int_mv *cur_mv,
const int mi_row, const int mi_col,
@@ -8001,11 +8013,22 @@
(void)args;
if (is_comp_pred) {
- if (this_mode == NEW_NEWMV) {
- cur_mv[0].as_int = args->single_newmv[ref_mv_idx][refs[0]].as_int;
- cur_mv[1].as_int = args->single_newmv[ref_mv_idx][refs[1]].as_int;
+ const int valid_mv0 = args->single_newmv_valid[ref_mv_idx][refs[0]];
+ const int valid_mv1 = args->single_newmv_valid[ref_mv_idx][refs[1]];
- if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
+ if (this_mode == NEW_NEWMV) {
+ if (valid_mv0) {
+ cur_mv[0].as_int = args->single_newmv[ref_mv_idx][refs[0]].as_int;
+ clamp_mv_in_range(x, &cur_mv[0], 0);
+ }
+ if (valid_mv1) {
+ cur_mv[1].as_int = args->single_newmv[ref_mv_idx][refs[1]].as_int;
+ clamp_mv_in_range(x, &cur_mv[1], 1);
+ }
+
+ // aomenc1
+ if (cpi->sf.comp_inter_joint_search_thresh <= bsize || !valid_mv0 ||
+ !valid_mv1) {
joint_motion_search(cpi, x, bsize, cur_mv, mi_row, mi_col, NULL, NULL,
0, rate_mv, 0);
} else {
@@ -8018,8 +8041,13 @@
}
}
} else if (this_mode == NEAREST_NEWMV || this_mode == NEAR_NEWMV) {
- cur_mv[1].as_int = args->single_newmv[ref_mv_idx][refs[1]].as_int;
- if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
+ if (valid_mv1) {
+ cur_mv[1].as_int = args->single_newmv[ref_mv_idx][refs[1]].as_int;
+ clamp_mv_in_range(x, &cur_mv[1], 1);
+ }
+
+ // aomenc2
+ if (cpi->sf.comp_inter_joint_search_thresh <= bsize || !valid_mv1) {
compound_single_motion_search_interinter(
cpi, x, bsize, cur_mv, mi_row, mi_col, NULL, 0, rate_mv, 0, 1);
} else {
@@ -8030,8 +8058,13 @@
}
} else {
assert(this_mode == NEW_NEARESTMV || this_mode == NEW_NEARMV);
- cur_mv[0].as_int = args->single_newmv[ref_mv_idx][refs[0]].as_int;
- if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
+ if (valid_mv0) {
+ cur_mv[0].as_int = args->single_newmv[ref_mv_idx][refs[0]].as_int;
+ clamp_mv_in_range(x, &cur_mv[0], 0);
+ }
+
+ // aomenc3
+ if (cpi->sf.comp_inter_joint_search_thresh <= bsize || !valid_mv0) {
compound_single_motion_search_interinter(
cpi, x, bsize, cur_mv, mi_row, mi_col, NULL, 0, rate_mv, 0, 0);
} else {
@@ -10349,7 +10382,8 @@
int this_rate_mv = 0;
for (i = 0; i < ref_mv_idx; ++i) {
// Check if the motion search result same as previous results
- if (cur_mv[0].as_int == args->single_newmv[i][refs[0]].as_int) {
+ if (cur_mv[0].as_int == args->single_newmv[i][refs[0]].as_int &&
+ args->single_newmv_valid[i][refs[0]]) {
// If the compared mode has no valid rd, it is unlikely this
// mode will be the best mode
if (mode_info[i].rd == INT64_MAX) {
