Stabilize temporal filtering and TPL motion search for HBD

Favor zero motion vector (0,0) in stationary/near-stationary regions
during temporal filtering motion search and TPL mode estimation.
Disable sub-block motion estimation in temporal filtering and clamp
filter strength to 0 for low-contrast blocks when sharpness is 3.

Bug: 548701298
Change-Id: Id5f1a8ed67a1b609159a15a969064419b053beab

av1/encoder/temporal_filter.c

diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index be7ff38..97be8fc 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -382,6 +382,27 @@
     *is_dc_diff_large = 50 * error < sse;
     if (src_var <= 2 * (int64_t)distortion) *is_low_cntras = 1;
 
+#if CONFIG_AV1_HIGHBITDEPTH
+    const MACROBLOCKD *xd = &mb->e_mbd;
+    if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) {
+      // Favor zero motion vector (0,0) for stationary/near-stationary regions
+      // when zero-motion error is close to the best found error or sufficiently
+      // small.
+      unsigned int zero_sse;
+      unsigned int zero_error = cpi->ppi->fn_ptr[block_size].vf(
+          ref_frame->y_buffer + y_offset, y_stride,
+          frame_to_filter->y_buffer + y_offset, y_stride, &zero_sse);
+      if (zero_error <= error + (error >> 4) || (zero_error / mb_pels) < 16) {
+        best_mv.as_mv = kZeroMv;
+        block_mv = kZeroMv;
+        *ref_mv = kZeroMv;
+        error = zero_error;
+        distortion = (int)zero_error;
+        block_mse = DIVIDE_AND_ROUND(error, mb_pels);
+      }
+    }
+#endif
+
     // On 4 mid-blocks in the 64x64 tf block.
     if (allow_me_for_sub_blks) {
       // midblock_size is 32x32, which corresponds to each 32x32 block in the
@@ -1092,7 +1113,12 @@
     // partition only if the difference between max and min source variance of
     // 4x4 blocks is greater than a threshold (which is derived empirically).
     bool allow_me_for_sub_blks = true;
-    if (cpi->sf.hl_sf.allow_sub_blk_me_in_tf) {
+    // In high bit-depth sharpness mode, disable sub-block motion estimation to
+    // avoid inconsistent filtering across sub-partitions and preserve
+    // sharpness.
+    if (mbd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) {
+      allow_me_for_sub_blks = false;
+    } else if (cpi->sf.hl_sf.allow_sub_blk_me_in_tf) {
       const int is_hbd = is_frame_high_bitdepth(frame_to_filter);
       // Initialize minimum variance to a large value and maximum variance to 0.
       double blk_4x4_var_min = DBL_MAX;
@@ -1139,7 +1165,7 @@
         filter_strength = AOMMIN(filter_strength, 1);
 
       if (cpi->oxcf.algo_cfg.sharpness == 3 && is_low_cntras)
-        filter_strength = AOMMIN(filter_strength, 3);
+        filter_strength = AOMMIN(filter_strength, mbd->bd > 8 ? 0 : 3);
 
       // Perform weighted averaging.
       if (frame == filter_frame_idx) {  // Frame to be filtered.

av1/encoder/tpl_model.c

diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index a22341b..34cf2bf 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -854,6 +854,20 @@
           cpi, x, src_mb_buffer, ref_mb, src_stride, ref_stride, src_width,
           ref_width, bsize, center_mvs[idx].mv.as_mv, &this_mv);
 
+#if CONFIG_AV1_HIGHBITDEPTH
+      if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) {
+        // Bias towards (0,0) / short motion vectors in stationary background.
+        // Require a noticeable SAD improvement before replacing (0,0) with a
+        // non-zero MV.
+        if (bestsme != UINT32_MAX && best_rfidx_mv.as_int == 0 &&
+            this_mv.as_int != 0) {
+          if (thissme + (thissme >> 4) >= bestsme) {
+            continue;
+          }
+        }
+      }
+#endif
+
       if (thissme < bestsme) {
         bestsme = thissme;
         best_rfidx_mv = this_mv;