commit 353b4b590472826d58b188b3e5b281c7f12d1400
author Jingning Han <jingning@google.com> Mon Feb 27 15:46:02 2023 -0800
committer Urvang Joshi <urvang@google.com> Tue Mar 28 05:35:52 2023 +0000
tree cbe0026b150ceaf43a54ba86f8dc5a7c797ac524
parent 2b1c4eb51aa05df5514e5cb8f37e50d15e8fb534

Check the motion search bound

(Ported from libaom commit 68047e7)

Add a secondary check to ensure that the lower bound is always
no greater than the upper bound in the motion search range. This
resolves an encoding failure under extreme frame size.

av1/encoder/mcomp.c

diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index 479b0aa..c1e2379 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -391,6 +391,9 @@
   if (mv_limits->col_max > col_max) mv_limits->col_max = col_max;
   if (mv_limits->row_min < row_min) mv_limits->row_min = row_min;
   if (mv_limits->row_max > row_max) mv_limits->row_max = row_max;
+
+  mv_limits->col_max = AOMMAX(mv_limits->col_min, mv_limits->col_max);
+  mv_limits->row_max = AOMMAX(mv_limits->row_min, mv_limits->row_max);
 }
 
 #if CONFIG_TIP

av1/encoder/mcomp.h

diff --git a/av1/encoder/mcomp.h b/av1/encoder/mcomp.h
index 635bfd9..e6571b2 100644
--- a/av1/encoder/mcomp.h
+++ b/av1/encoder/mcomp.h
@@ -580,10 +580,13 @@
   const int mv_low = av1_lower_mv_limit(MV_LOW + 1, sub_pel_prec_shift);
   const int mv_upp = av1_lower_mv_limit(MV_UPP - 1, sub_pel_prec_shift);
 
-  const int minc = AOMMAX(col_min, low_prec_ref_mv.col - max_mv);
-  const int maxc = AOMMIN(col_max, low_prec_ref_mv.col + max_mv);
-  const int minr = AOMMAX(row_min, low_prec_ref_mv.row - max_mv);
-  const int maxr = AOMMIN(row_max, low_prec_ref_mv.row + max_mv);
+  int minc = AOMMAX(col_min, low_prec_ref_mv.col - max_mv);
+  int maxc = AOMMIN(col_max, low_prec_ref_mv.col + max_mv);
+  int minr = AOMMAX(row_min, low_prec_ref_mv.row - max_mv);
+  int maxr = AOMMIN(row_max, low_prec_ref_mv.row + max_mv);
+
+  maxc = AOMMAX(minc, maxc);
+  maxr = AOMMAX(minr, maxr);
 
   subpel_limits->col_min = AOMMAX(mv_low + (1 << sub_pel_prec_shift), minc);
   subpel_limits->col_max = AOMMIN(mv_upp - (1 << sub_pel_prec_shift), maxc);
@@ -592,14 +595,14 @@
 #else
 
   const int max_mv = GET_MV_SUBPEL(MAX_FULL_PEL_VAL);
-  const int minc =
-      AOMMAX(GET_MV_SUBPEL(mv_limits->col_min), ref_mv->col - max_mv);
-  const int maxc =
-      AOMMIN(GET_MV_SUBPEL(mv_limits->col_max), ref_mv->col + max_mv);
-  const int minr =
-      AOMMAX(GET_MV_SUBPEL(mv_limits->row_min), ref_mv->row - max_mv);
-  const int maxr =
-      AOMMIN(GET_MV_SUBPEL(mv_limits->row_max), ref_mv->row + max_mv);
+  int minc = AOMMAX(GET_MV_SUBPEL(mv_limits->col_min), ref_mv->col - max_mv);
+  int maxc = AOMMIN(GET_MV_SUBPEL(mv_limits->col_max), ref_mv->col + max_mv);
+  int minr = AOMMAX(GET_MV_SUBPEL(mv_limits->row_min), ref_mv->row - max_mv);
+  int maxr = AOMMIN(GET_MV_SUBPEL(mv_limits->row_max), ref_mv->row + max_mv);
+
+  maxc = AOMMAX(minc, maxc);
+  maxr = AOMMAX(minr, maxr);
+
   subpel_limits->col_min = AOMMAX(MV_LOW + 1, minc);
   subpel_limits->col_max = AOMMIN(MV_UPP - 1, maxc);
   subpel_limits->row_min = AOMMAX(MV_LOW + 1, minr);