rtc: Clamp forced NEWMV vectors in handle_inter_mode_nonrd

In handle_inter_mode_nonrd(), when NEWMV reuses the superblock motion
vector (x->sb_me_mv, searched relative to (0, 0)) or an inter-layer SVC
motion vector (force_mv_inter_layer), search_new_mv() is bypassed. As a
result, the motion vector was not constrained relative to the block's
reference MV predictor (ref_mv), allowing |mv - ref_mv| to exceed MV_MAX
(MV_CLASS_10) during bitstream encoding, and leaving rate_mv at 0.

Clamp the forced NEWMV motion vector to SubpelMvLimits around ref_mv,
skip NEWMV if the clamped vector equals ref_mv, and compute rate_mv
via av1_mv_bit_cost().

Unittest added.

Bug: 574032795
Change-Id: Ib688ce9b4a852a12ba755a281ab532332ffa6126

av1/encoder/nonrd_pickmode.c

diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 0034f12..7c4f32e 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -2963,6 +2963,24 @@
     }
   }
 
+  if (this_mode == NEWMV &&
+      (force_mv_inter_layer || (x->sb_me_block && ref_frame == LAST_FRAME))) {
+    // When NEWMV reuses the superblock ME vector (x->sb_me_mv, searched
+    // relative to (0, 0)) or an inter-layer SVC motion vector, search_new_mv()
+    // is bypassed. Clamp the motion vector to the valid subpel search range
+    // relative to the block's reference MV predictor (ref_mv) so that the MV
+    // delta stays within [MV_LOW, MV_UPP], skip NEWMV if the clamped vector
+    // equals ref_mv, and compute its MV bit cost.
+    const int_mv ref_mv = av1_get_ref_mv(x, 0);
+    SubpelMvLimits mv_limits;
+    av1_set_subpel_mv_search_range(&mv_limits, &x->mv_limits, &ref_mv.as_mv);
+    clamp_mv(&this_mv->as_mv, &mv_limits);
+    if (this_mv->as_int == ref_mv.as_int) return true;
+    rate_mv = av1_mv_bit_cost(&this_mv->as_mv, &ref_mv.as_mv,
+                              x->mv_costs->nmv_joint_cost,
+                              x->mv_costs->mv_cost_stack, MV_COST_WEIGHT);
+  }
+
   // Check the current motion vector is same as that of previously evaluated
   // motion vectors.
   for (PREDICTION_MODE inter_mv_mode = NEARESTMV; inter_mv_mode <= NEWMV;

test/encode_api_test.cc

diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 352a51c..eac1ec0 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -4376,4 +4376,99 @@
 }
 #endif  // CONFIG_AV1_DECODER
 
+// Verify that when handle_inter_mode_nonrd() reuses the superblock motion
+// vector (x->sb_me_mv, searched relative to zero) for NEWMV on screen content,
+// the vector is clamped to the valid subpel range relative to the block's
+// reference MV predictor (ref_mv) so that |mv - ref_mv| does not exceed MV_MAX.
+TEST(EncodeAPI, NonrdSbMeForcedNewmvRange) {
+  // At >= 4K resolution (width * height >= 3840 * 2160),
+  // do_int_pro_motion_estimation() doubles the screen-content search window
+  // from 512 to 1024 full-pel pixels ([-1024, +1023] relative to (0, 0)).
+  // Use a portrait 2160x3840 frame so +/-1023 pixel vertical displacements
+  // stay well within the frame height.
+  constexpr int kWidth = 2160;
+  constexpr int kHeight = 3840;
+  constexpr int kActiveW = 128;
+  constexpr int kActiveH = 2312;
+
+  aom_codec_iface_t *const iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+            AOM_CODEC_OK);
+  cfg.g_w = kWidth;
+  cfg.g_h = kHeight;
+  cfg.g_lag_in_frames = 0;
+  cfg.g_threads = 1;
+  cfg.rc_min_quantizer = 1;
+  cfg.rc_max_quantizer = 1;
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 7), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN),
+            AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_QUANTIZER_ONE_PASS, 1),
+            AOM_CODEC_OK);
+
+  aom_image_t *img0 =
+      aom_img_alloc(nullptr, AOM_IMG_FMT_I420, kWidth, kHeight, 1);
+  aom_image_t *img1 =
+      aom_img_alloc(nullptr, AOM_IMG_FMT_I420, kWidth, kHeight, 1);
+  ASSERT_NE(img0, nullptr);
+  ASSERT_NE(img1, nullptr);
+  FillImage(img0, 128);
+  FillImage(img1, 128);
+
+  // Fill a 128-px wide strip (one superblock column) of the keyframe (img0)
+  // with random luma values so that displaced copies in img1 have a unique
+  // zero-SAD match and a large zero-MV SAD.
+  ::libaom_test::ACMRandom rnd(::libaom_test::ACMRandom::DeterministicSeed());
+  for (int y = 0; y < kActiveH; ++y) {
+    uint8_t *const row0 = img0->planes[AOM_PLANE_Y] + y * img0->stride[0];
+    for (int x = 0; x < kActiveW; ++x) {
+      row0[x] = rnd.Rand8();
+    }
+  }
+
+  // Construct three vertically adjacent superblock regions in img1 (delta
+  // frame) to produce an unrepresentable MV difference if x->sb_me_mv is not
+  // clamped relative to ref_mv:
+  //
+  // 1. y in [0, 1152): copied from img0 at y + 1023 (dy = +1023 px).
+  //    do_int_pro_motion_estimation() searches [-1024, +1023] relative to
+  //    (0, 0), finds the exact match, and sets x->sb_me_mv = (+8184, 0)
+  //    (+1023 px in 1/8-pel units).
+  //
+  // 2. y in [1152, 1280): copied from img0 at y + 1031 (dy = +1031 px).
+  //    Because +1031 px is outside int_pro's [-1024, +1023] window around
+  //    (0, 0), x->sb_me_block is not set. Instead, normal NEWMV motion search
+  //    (search_new_mv()) starts from the top neighbor's predictor
+  //    ref_mv = (+8184, 0) (+1023 px) and steps +8 px to find the exact match
+  //    at mv = (+8248, 0) (+1031 px).
+  //
+  // 3. y in [1280, 1408): copied from img0 at y - 1023 (dy = -1023 px).
+  //    At y >= 1280, -1023 px upward is inside img0, so int_pro ME finds the
+  //    exact match relative to (0, 0) and sets x->sb_me_block = 1 with
+  //    x->sb_me_mv = (-8184, 0) (-1023 px). However, this superblock's coding
+  //    predictor from its top neighbor (region 2) is ref_mv = (+8248, 0)
+  //    (+1031 px). Without clamping x->sb_me_mv relative to ref_mv in
+  //    handle_inter_mode_nonrd(), NEWMV encodes the difference:
+  //      mv_diff.row = -8184 - 8248 = -16432 (-2054 px)
+  //    which exceeds MV_MAX (16383, +/-2047.875 px) and triggers
+  //    Assertion `c <= MV_CLASS_10' failed in av1_get_mv_class().
+  for (int y = 0; y < 1408; ++y) {
+    const int dy = (y < 1152) ? 1023 : (y < 1280) ? 1031 : -1023;
+    memcpy(img1->planes[AOM_PLANE_Y] + y * img1->stride[0],
+           img0->planes[AOM_PLANE_Y] + (y + dy) * img0->stride[0], kActiveW);
+  }
+
+  ASSERT_EQ(aom_codec_encode(&enc, img0, 0, 1, AOM_EFLAG_FORCE_KF),
+            AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_encode(&enc, img1, 1, 1, 0), AOM_CODEC_OK);
+
+  aom_img_free(img1);
+  aom_img_free(img0);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
 }  // namespace