Validate tpl frame index before using it
Check if the tpl frame index is < MAX_TPL_FRAME_IDX before using it as
the index for the cpi->ppi->tpl_data.tpl_frame[] array.
BUG=aomedia:3054
Change-Id: Ie5593e0c0baad4d655eb169a0e3fb1049629ce7f
(cherry picked from commit d43470b76bc2dceb43c0dbd00dddfffb72df77bf)
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 35305ed..e431448 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -331,14 +331,14 @@
MACROBLOCK *x = &td->mb;
const int frame_idx = cpi->gf_frame_index;
TplParams *const tpl_data = &cpi->tpl_data;
- TplDepFrame *tpl_frame = &tpl_data->tpl_frame[frame_idx];
const uint8_t block_mis_log2 = tpl_data->tpl_stats_block_mis_log2;
av1_zero(x->tpl_keep_ref_frame);
+ if (frame_idx >= MAX_TPL_FRAME_IDX) return;
+ TplDepFrame *tpl_frame = &tpl_data->tpl_frame[frame_idx];
if (tpl_frame->is_valid == 0) return;
if (!is_frame_tpl_eligible(gf_group, cpi->gf_frame_index)) return;
- if (frame_idx >= MAX_TPL_FRAME_IDX) return;
if (cpi->oxcf.q_cfg.aq_mode != NO_AQ) return;
const int is_overlay = cpi->gf_group.update_type[frame_idx] == OVERLAY_UPDATE;
diff --git a/av1/encoder/encodeframe_utils.c b/av1/encoder/encodeframe_utils.c
index 27f306f..c9b1afb 100644
--- a/av1/encoder/encodeframe_utils.c
+++ b/av1/encoder/encodeframe_utils.c
@@ -90,12 +90,12 @@
assert(IMPLIES(cpi->gf_group.size > 0,
cpi->gf_frame_index < cpi->gf_group.size));
const int tpl_idx = cpi->gf_frame_index;
- const TplDepFrame *tpl_frame = &cpi->tpl_data.tpl_frame[tpl_idx];
const int deltaq_rdmult = set_deltaq_rdmult(cpi, x);
+ if (tpl_idx >= MAX_TPL_FRAME_IDX) return deltaq_rdmult;
+ const TplDepFrame *tpl_frame = &cpi->tpl_data.tpl_frame[tpl_idx];
if (tpl_frame->is_valid == 0) return deltaq_rdmult;
if (!is_frame_tpl_eligible(gf_group, cpi->gf_frame_index))
return deltaq_rdmult;
- if (tpl_idx >= MAX_TPL_FRAME_IDX) return deltaq_rdmult;
if (cpi->oxcf.q_cfg.aq_mode != NO_AQ) return deltaq_rdmult;
const int mi_col_sr =
@@ -688,21 +688,21 @@
cpi->gf_frame_index < cpi->gf_group.size));
const int tpl_idx = cpi->gf_frame_index;
TplParams *const tpl_data = &cpi->tpl_data;
- TplDepFrame *tpl_frame = &tpl_data->tpl_frame[tpl_idx];
- TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
const uint8_t block_mis_log2 = tpl_data->tpl_stats_block_mis_log2;
- int tpl_stride = tpl_frame->stride;
int64_t intra_cost = 0;
int64_t mc_dep_cost = 0;
const int mi_wide = mi_size_wide[bsize];
const int mi_high = mi_size_high[bsize];
+ if (tpl_idx >= MAX_TPL_FRAME_IDX) return orig_rdmult;
+
+ TplDepFrame *tpl_frame = &tpl_data->tpl_frame[tpl_idx];
+ TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
+ int tpl_stride = tpl_frame->stride;
if (tpl_frame->is_valid == 0) return orig_rdmult;
if (!is_frame_tpl_eligible(gf_group, cpi->gf_frame_index)) return orig_rdmult;
- if (cpi->gf_frame_index >= MAX_TPL_FRAME_IDX) return orig_rdmult;
-
int mi_count = 0;
const int mi_col_sr =
coded_to_superres_mi(mi_col, cm->superres_scale_denominator);
@@ -825,15 +825,16 @@
AV1_COMMON *const cm = &cpi->common;
const int gf_group_index = cpi->gf_frame_index;
TplParams *const tpl_data = &cpi->tpl_data;
- TplDepFrame *tpl_frame = &tpl_data->tpl_frame[gf_group_index];
- TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
- int tpl_stride = tpl_frame->stride;
const int mi_wide = mi_size_wide[bsize];
const int mi_high = mi_size_high[bsize];
- if (tpl_frame->is_valid == 0) return;
if (gf_group_index >= MAX_TPL_FRAME_IDX) return;
+ TplDepFrame *tpl_frame = &tpl_data->tpl_frame[gf_group_index];
+ TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
+ int tpl_stride = tpl_frame->stride;
+ if (tpl_frame->is_valid == 0) return;
+
int mi_count = 0;
int count = 0;
const int mi_col_sr =
@@ -896,22 +897,22 @@
cpi->gf_frame_index < cpi->gf_group.size));
const int tpl_idx = cpi->gf_frame_index;
TplParams *const tpl_data = &cpi->tpl_data;
- TplDepFrame *tpl_frame = &tpl_data->tpl_frame[tpl_idx];
- TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
const uint8_t block_mis_log2 = tpl_data->tpl_stats_block_mis_log2;
- int tpl_stride = tpl_frame->stride;
int64_t intra_cost = 0;
int64_t mc_dep_cost = 0;
const int mi_wide = mi_size_wide[bsize];
const int mi_high = mi_size_high[bsize];
const int base_qindex = cm->quant_params.base_qindex;
+ if (tpl_idx >= MAX_TPL_FRAME_IDX) return base_qindex;
+
+ TplDepFrame *tpl_frame = &tpl_data->tpl_frame[tpl_idx];
+ TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
+ int tpl_stride = tpl_frame->stride;
if (tpl_frame->is_valid == 0) return base_qindex;
if (!is_frame_tpl_eligible(gf_group, cpi->gf_frame_index)) return base_qindex;
- if (cpi->gf_frame_index >= MAX_TPL_FRAME_IDX) return base_qindex;
-
int mi_count = 0;
const int mi_col_sr =
coded_to_superres_mi(mi_col, cm->superres_scale_denominator);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index a695335..f462b65 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2178,8 +2178,11 @@
cpi->gf_frame_index < cpi->gf_group.size));
const int tpl_idx = cpi->gf_frame_index;
TplParams *const tpl_data = &cpi->tpl_data;
+ if (tpl_idx >= MAX_TPL_FRAME_IDX) {
+ return;
+ }
const TplDepFrame *tpl_frame = &tpl_data->tpl_frame[tpl_idx];
- if (tpl_idx >= MAX_TPL_FRAME_IDX || !tpl_frame->is_valid) {
+ if (!tpl_frame->is_valid) {
return;
}
@@ -2591,10 +2594,10 @@
const PREDICTION_MODE this_mode = mbmi->mode;
const int tpl_idx = cpi->gf_frame_index;
- TplDepFrame *tpl_frame = &cpi->tpl_data.tpl_frame[tpl_idx];
+ TplParams *const tpl_data = &cpi->tpl_data;
const int prune_modes_based_on_tpl =
cpi->sf.inter_sf.prune_inter_modes_based_on_tpl &&
- tpl_idx < MAX_TPL_FRAME_IDX && tpl_frame->is_valid;
+ tpl_idx < MAX_TPL_FRAME_IDX && tpl_data->tpl_frame[tpl_idx].is_valid;
int i;
// Reference frames for this mode
const int refs[2] = { mbmi->ref_frame[0],
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index 2643665..2cf5a18 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -1661,11 +1661,11 @@
assert(IMPLIES(cpi->gf_group.size > 0,
cpi->gf_frame_index < cpi->gf_group.size));
const int tpl_idx = cpi->gf_frame_index;
- TplDepFrame *tpl_frame = &cpi->tpl_data.tpl_frame[tpl_idx];
+ if (tpl_idx >= MAX_TPL_FRAME_IDX) return;
+ TplDepFrame *tpl_frame = &cpi->tpl_data.tpl_frame[tpl_idx];
if (tpl_frame->is_valid == 0) return;
if (!is_frame_tpl_eligible(gf_group, cpi->gf_frame_index)) return;
- if (tpl_idx >= MAX_TPL_FRAME_IDX) return;
if (cpi->oxcf.q_cfg.aq_mode != NO_AQ) return;
const int mi_col_sr =
