Fix seg fault when --enable-keyframe-filtering=2 enable_keyframe_filtering > 1 will introduce an extra overlay frame at key frame location. However when baseline_gf_interval == MAX_STATIC_GF_GROUP_LENGTH, we can't afford to have an extra overlay frame. Otherwise, the gf_group->size will exceed MAX_STATIC_GF_GROUP_LENGTH, which leads to memory error. A cheap solution is to turn off kf_decomp when baseline_gf_interval == MAX_STATIC_GF_GROUP_LENGTH. BUG=aomedia:3163 Change-Id: I05c604065496d698611f58ceb5c6ac7c7bbf5ebe
diff --git a/av1/encoder/gop_structure.c b/av1/encoder/gop_structure.c index 70c1a4f..046fbd9 100644 --- a/av1/encoder/gop_structure.c +++ b/av1/encoder/gop_structure.c
@@ -545,9 +545,11 @@ static int construct_multi_layer_gf_structure( AV1_COMP *cpi, TWO_PASS *twopass, GF_GROUP *const gf_group, - RATE_CONTROL *rc, FRAME_INFO *const frame_info, int gf_interval, + RATE_CONTROL *rc, FRAME_INFO *const frame_info, int baseline_gf_interval, FRAME_UPDATE_TYPE first_frame_update_type) { PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc; + // TODO(angiebird): Why do we need "-1" here? + const int gf_interval = baseline_gf_interval - 1; int frame_index = 0; int cur_frame_index = 0; @@ -581,7 +583,18 @@ #endif // CONFIG_FRAME_PARALLEL_ENCODE_2 #endif // CONFIG_FRAME_PARALLEL_ENCODE - const int kf_decomp = cpi->oxcf.kf_cfg.enable_keyframe_filtering > 1; + int kf_decomp = cpi->oxcf.kf_cfg.enable_keyframe_filtering > 1; + // This is a patch that fixes https://crbug.com/aomedia/3163 + // enable_keyframe_filtering > 1 will introduce an extra overlay frame at + // key frame location. However when + // baseline_gf_interval == MAX_STATIC_GF_GROUP_LENGTH, we can't + // afford to have an extra overlay frame. Otherwise, the gf_group->size will + // become MAX_STATIC_GF_GROUP_LENGTH + 1, which causes memory error. + // A cheap solution is to turn of kf_decomp here. + // TODO(angiebird): Find a systematic way to solve this issue. + if (baseline_gf_interval == MAX_STATIC_GF_GROUP_LENGTH) { + kf_decomp = 0; + } if (first_frame_update_type == KF_UPDATE) { gf_group->update_type[frame_index] = kf_decomp ? ARF_UPDATE : KF_UPDATE; gf_group->arf_src_offset[frame_index] = 0; @@ -876,7 +889,7 @@ first_frame_update_type = GF_UPDATE; gf_group->size = construct_multi_layer_gf_structure( - cpi, twopass, gf_group, rc, frame_info, p_rc->baseline_gf_interval - 1, + cpi, twopass, gf_group, rc, frame_info, p_rc->baseline_gf_interval, first_frame_update_type); if (gf_group->max_layer_depth_allowed == 0)