Smooth 2-pass rate control response and bit redistribution

Allow over_shoot_pct <= 100 and unclamp pct_error for HBD sharpness 3
mode to adapt more smoothly to high bitrate peaks.
Suppress fast extra bit redistribution when currently overshooting
to prevent inter-frame QP oscillations.

Bug: 548701298
Change-Id: Iacdd96a988ea3ad97eae89a32400659d8f3e5794

av1/encoder/pass2_strategy.c

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 36775ff..42e9186 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -4600,13 +4600,12 @@
       }
       twopass->extend_maxq -= 1;
       // Overshoot
-    } else if ((rc_cfg->over_shoot_pct < 100) &&
+    } else if ((rc_cfg->over_shoot_pct <= 100) &&
                (p_rc->rolling_actual_bits > p_rc->rolling_target_bits)) {
       int pct_error =
           ((p_rc->rolling_actual_bits - p_rc->rolling_target_bits) * 100) /
           p_rc->rolling_target_bits;
 
-      pct_error = clamp(pct_error, 0, 100);
       if ((pct_error >= rc_cfg->over_shoot_pct) &&
           (p_rc->rate_error_estimate < 0)) {
         twopass->extend_maxq += 1;
@@ -4632,13 +4631,21 @@
     // frame is unexpectedly almost perfectly predicted by the ARF or GF
     // but not very well predcited by the previous frame.
     if (!frame_is_kf_gf_arf(cpi) && !cpi->rc.is_src_frame_alt_ref) {
-      int fast_extra_thresh = rc->base_frame_target / HIGH_UNDERSHOOT_RATIO;
-      if (rc->projected_frame_size < fast_extra_thresh) {
-        p_rc->vbr_bits_off_target_fast +=
-            fast_extra_thresh - rc->projected_frame_size;
-        p_rc->vbr_bits_off_target_fast =
-            AOMMIN(p_rc->vbr_bits_off_target_fast,
-                   (4 * (int64_t)rc->avg_frame_bandwidth));
+#if CONFIG_AV1_HIGHBITDEPTH
+      if (cpi->common.seq_params->bit_depth > 8 &&
+          cpi->oxcf.algo_cfg.sharpness == 3 && p_rc->vbr_bits_off_target < 0) {
+        p_rc->vbr_bits_off_target_fast = 0;
+      } else
+#endif
+      {
+        int fast_extra_thresh = rc->base_frame_target / HIGH_UNDERSHOOT_RATIO;
+        if (rc->projected_frame_size < fast_extra_thresh) {
+          p_rc->vbr_bits_off_target_fast +=
+              fast_extra_thresh - rc->projected_frame_size;
+          p_rc->vbr_bits_off_target_fast =
+              AOMMIN(p_rc->vbr_bits_off_target_fast,
+                     (4 * (int64_t)rc->avg_frame_bandwidth));
+        }
       }
     }
 

av1/encoder/ratectrl.c

diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 677432f..03cb5c9 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2789,14 +2789,18 @@
       simulate_parallel_frame ? cpi->ppi->p_rc.temp_vbr_bits_off_target_fast
                               : p_rc->vbr_bits_off_target_fast;
 #endif
-  // Fast redistribution of bits arising from massive local undershoot.
-  // Don't do it for kf,arf,gf or overlay frames.
   if (!frame_is_kf_gf_arf(cpi) &&
 #if CONFIG_FPMT_TEST
       vbr_bits_off_target_fast &&
 #else
       p_rc->vbr_bits_off_target_fast &&
 #endif
+#if CONFIG_AV1_HIGHBITDEPTH
+      (cpi->common.seq_params->bit_depth > 8 &&
+               cpi->oxcf.algo_cfg.sharpness == 3
+           ? vbr_bits_off_target >= 0
+           : 1) &&
+#endif
       !rc->is_src_frame_alt_ref) {
     int64_t one_frame_bits = AOMMAX(rc->avg_frame_bandwidth, frame_target);
     int64_t fast_extra_bits;