commit d25c09124223baf98dc79ad82a92be7bb704112d
author chiyotsai <chiyotsai@google.com> Mon Mar 30 16:26:57 2020 -0700
committer Chi Yo Tsai <chiyotsai@google.com> Wed Apr 01 06:38:27 2020 +0000
tree ff9604bbc8ee0642adc5f5cda3bfa68fee7505f8
parent c7554aaf5d8b75a2faec3bfe18a8465175314bde

Fix a segfault in prune_inter_modes_based_on_tpl

Current TPL has a maximum size of MAX_LENGTH_TPL_FRAME_STATS = 79, but
the gf group size can be larger than that. As a result, the speed
feature prune_inter_modes_based_on_tpl can have an out of bound array
access and seg fault.

This CL adds a check to avoid the OOB access.

BUG=aomedia:2623

Change-Id: I35de168673ccfc80bb9d2f13eb3ce7fe50afdbe7

av1/encoder/encoder.h

diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index e1d7c24..452fcca 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -150,7 +150,8 @@
 } UENUM1BYTE(SS_CFG_OFFSET);
 
 // TODO(jingning): This needs to be cleaned up next.
-#define MAX_LENGTH_TPL_FRAME_STATS (70 + 9)
+#define TPL_MAX_LENGTH (70)
+#define MAX_LENGTH_TPL_FRAME_STATS (TPL_MAX_LENGTH + REF_FRAMES + 1)
 
 typedef struct TplDepStats {
   int64_t intra_cost;

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 3a0ea59..843931d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2120,7 +2120,9 @@
   assert(IMPLIES(gf_group->size > 0, gf_group->index < gf_group->size));
   const int tpl_idx = gf_group->index;
   const TplDepFrame *tpl_frame = &cpi->tpl_frame[tpl_idx];
-  if (!tpl_frame->is_valid) return;
+  if (tpl_idx >= TPL_MAX_LENGTH || !tpl_frame->is_valid) {
+    return;
+  }
 
   const TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
   const int mi_wide = mi_size_wide[bsize];
@@ -2231,7 +2233,8 @@
   const int tpl_idx = gf_group->index;
   TplDepFrame *tpl_frame = &cpi->tpl_frame[tpl_idx];
   const int prune_modes_based_on_tpl =
-      cpi->sf.inter_sf.prune_inter_modes_based_on_tpl && tpl_frame->is_valid;
+      cpi->sf.inter_sf.prune_inter_modes_based_on_tpl &&
+      tpl_idx >= TPL_MAX_LENGTH && tpl_frame->is_valid;
   int i;
   const int refs[2] = { mbmi->ref_frame[0],
                         (mbmi->ref_frame[1] < 0 ? 0 : mbmi->ref_frame[1]) };