Handle buffer pointer in LAP mode to avoid overflow

1. Always allocate at least MAX_GF_LENGTH_LAP + 1 stats buffers
even though lag-in-frames < MAX_GF_LENGTH_LAP.

2. Use compacting sliding window for stats buffers in LAP mode to avoid
out of boundary problems.

3. rest_frames in av1_get_second_pass_params is mistakenly calculated
in LAP mode. Correct the off by 1 error.

BUG=aomedia:504317456

Change-Id: I23df3262e760f9ae501434f0f828c0e745df73f4

av1/encoder/encoder.h

diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 61153b9..0e38bb8 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -4194,7 +4194,10 @@
 // Function return size of frame stats buffer
 static inline int get_stats_buf_size(int num_lap_buffer, int num_lag_buffer) {
   /* if lookahead is enabled return num_lap_buffers else num_lag_buffers */
-  return (num_lap_buffer > 0 ? num_lap_buffer + 1 : num_lag_buffer);
+  if (num_lap_buffer > 0) {
+    return AOMMAX(num_lap_buffer + 1, MAX_GF_LENGTH_LAP + 1);
+  }
+  return num_lag_buffer;
 }
 
 // TODO(zoeliu): To set up cpi->oxcf.gf_cfg.enable_auto_brf

av1/encoder/firstpass.c

diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index e8c15cb..605e968 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -1004,6 +1004,19 @@
          twopass->stats_buf_ctx->stats_in_buf_end)) {
       twopass->stats_buf_ctx->stats_in_end =
           twopass->stats_buf_ctx->stats_in_start;
+    } else if (cpi->ppi->lap_enabled &&
+               (twopass->stats_buf_ctx->stats_in_end >=
+                twopass->stats_buf_ctx->stats_in_buf_end)) {
+      const int num_valid = (int)(twopass->stats_buf_ctx->stats_in_end -
+                                  cpi->twopass_frame.stats_in);
+      if (num_valid > 0) {
+        memmove(twopass->stats_buf_ctx->stats_in_start,
+                cpi->twopass_frame.stats_in,
+                num_valid * sizeof(FIRSTPASS_STATS));
+      }
+      cpi->twopass_frame.stats_in = twopass->stats_buf_ctx->stats_in_start;
+      twopass->stats_buf_ctx->stats_in_end =
+          twopass->stats_buf_ctx->stats_in_start + num_valid;
     }
   }
 }

av1/encoder/pass2_strategy.c

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 52b92b2..81adede 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -4100,10 +4100,12 @@
       // how many frames we can analyze from this frame
       int rest_frames =
           AOMMIN(rc->frames_to_key, MAX_FIRSTPASS_ANALYSIS_FRAMES);
-      rest_frames =
-          AOMMIN(rest_frames, (int)(twopass->stats_buf_ctx->stats_in_end -
-                                    cpi->twopass_frame.stats_in +
-                                    (rc->frames_since_key == 0)));
+      int available_frames = (int)(twopass->stats_buf_ctx->stats_in_end -
+                                   cpi->twopass_frame.stats_in);
+      if (!cpi->ppi->lap_enabled) {
+        available_frames += (rc->frames_since_key == 0);
+      }
+      rest_frames = AOMMIN(rest_frames, available_frames);
       p_rc->frames_till_regions_update = rest_frames;
 
       int ret;
@@ -4114,9 +4116,8 @@
                        twopass->stats_buf_ctx->stats_in_end, cpi->common.error);
         estimate_coeff(twopass->stats_buf_ctx->stats_in_start,
                        twopass->stats_buf_ctx->stats_in_end);
-        ret = identify_regions(cpi->twopass_frame.stats_in, rest_frames,
-                               (rc->frames_since_key == 0), p_rc->regions,
-                               &p_rc->num_regions);
+        ret = identify_regions(cpi->twopass_frame.stats_in, rest_frames, 0,
+                               p_rc->regions, &p_rc->num_regions);
       } else {
         ret = identify_regions(
             cpi->twopass_frame.stats_in - (rc->frames_since_key == 0),

test/encode_api_test.cc

diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 4b75ece..70f7c46 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -2445,4 +2445,47 @@
   ASSERT_EQ(aom_codec_destroy(&codec), AOM_CODEC_OK);
 }
 
+TEST(EncodeAPI, Buganizer504317456) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_GOOD_QUALITY),
+            AOM_CODEC_OK);
+
+  cfg.g_w = 640;
+  cfg.g_h = 360;
+  cfg.g_lag_in_frames = 1;
+  cfg.g_error_resilient = 0;
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+
+  aom_image_t *img = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 640, 360, 1);
+  FillImage(img, 128);
+  ASSERT_NE(img, nullptr);
+
+  struct Frame {
+    int64_t pts;
+    unsigned long duration;
+    aom_enc_frame_flags_t flags;
+  } frames[] = { { 0, 2, 0 },
+                 { 1075, 9, AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF },
+                 { 2099, 6, AOM_EFLAG_FORCE_KF },
+                 { 3035, 5, AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_FORCE_KF } };
+
+  for (const auto &f : frames) {
+    ASSERT_EQ(aom_codec_encode(&enc, img, f.pts, f.duration, f.flags),
+              AOM_CODEC_OK);
+  }
+
+  // Add 10+ additional frames to extend overflow and cause crash on destruction
+  for (int i = 0; i < 15; ++i) {
+    int64_t pts = 4000 + i * 100;
+    aom_enc_frame_flags_t flags = (i % 3 == 0) ? AOM_EFLAG_FORCE_KF : 0;
+    ASSERT_EQ(aom_codec_encode(&enc, img, pts, 5, flags), AOM_CODEC_OK);
+  }
+
+  aom_img_free(img);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
 }  // namespace