commit c92d48a356114a3ce871c348865850d44f6477b0
author bohanli <bohanli@google.com> Mon Mar 02 10:42:05 2020 -0800
committer Bohan Li <bohanli@google.com> Wed Mar 04 23:27:41 2020 +0000
tree 302c90cfede2a9aed70958091bd4e95aa83619ae
parent a0ea2b8a874b587a06d00235666caba7801a71ac

Fix an ASAN failure bug in calculate_gf_length.

BUG=aomedia:2595

Check end of stats_buf before reading it.

Change-Id: Ic9d1ce4f0bb639e09e0bef2eb67cd822fdc24ff1

av1/encoder/pass2_strategy.c

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 6a4e20c..67d19e8 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -1156,6 +1156,11 @@
         int after_pad =
             AOMMIN(MAX_PAD_GF_CHECK, rc->frames_to_key - cur_last - 1);
         for (n = cur_start - before_pad; n <= cur_last + after_pad; n++) {
+          if (start_pos + n - 1 > twopass->stats_buf_ctx->stats_in_end) {
+            after_pad = n - cur_last - 1;
+            assert(after_pad >= 0);
+            break;
+          }
           errs[n + before_pad - cur_start] = (start_pos + n - 1)->coded_error;
         }
         const int len = before_pad + after_pad + cur_last - cur_start + 1;

test/gf_pyr_height_test.cc

diff --git a/test/gf_pyr_height_test.cc b/test/gf_pyr_height_test.cc
index 0befcdf..b1ade67 100644
--- a/test/gf_pyr_height_test.cc
+++ b/test/gf_pyr_height_test.cc
@@ -26,13 +26,13 @@
 } kTestParams[] = {
   // gf_min_pyr_height = 0
   { 0, 0, 33.40 },
-  { 0, 1, 34.50 },
+  { 0, 1, 34.00 },
   { 0, 2, 34.00 },
   { 0, 3, 34.20 },
   { 0, 4, 34.30 },
   { 0, 5, 34.40 },
   // gf_min_pyr_height = 1
-  { 1, 1, 34.50 },
+  { 1, 1, 34.00 },
   { 1, 2, 34.00 },
   { 1, 3, 34.20 },
   { 1, 4, 34.30 },
@@ -140,7 +140,7 @@
   double psnr_;
 };
 
-TEST_P(GFPyrHeightTest, DISABLED_EncodeAndVerifyPSNR) {
+TEST_P(GFPyrHeightTest, EncodeAndVerifyPSNR) {
   libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
                                      cfg_.g_timebase.den, cfg_.g_timebase.num,
                                      0, 32);