Fix memory leak and fuzz issues
This CL fixes the memory leak and fuzz issues reported by
Jenkin test.
BUG=aomedia:3019
BUG=oss-fuzz:33030
Change-Id: Ice9fb68213e6b2a9eccceddb83b6f25f774e80f7
diff --git a/av1/common/alloccommon.c b/av1/common/alloccommon.c
index 5e0c9ce..d8594ed 100644
--- a/av1/common/alloccommon.c
+++ b/av1/common/alloccommon.c
@@ -63,27 +63,46 @@
void av1_alloc_cdef_linebuf(AV1_COMMON *const cm) {
const int num_planes = av1_num_planes(cm);
const int luma_stride = cm->mi_params.mi_cols << MI_SIZE_LOG2;
+ CdefInfo *cdef_info = &cm->cdef_info;
+ // Check for configuration change
+ const int is_sub_sampling_changed =
+ (cdef_info->allocated_subsampling_x != cm->seq_params.subsampling_x ||
+ cdef_info->allocated_subsampling_y != cm->seq_params.subsampling_y);
const int is_frame_scaled =
- (cm->cdef_info.allocated_mi_cols != cm->mi_params.mi_cols ||
- cm->cdef_info.allocated_mi_rows != cm->mi_params.mi_rows);
+ cdef_info->allocated_mi_cols != cm->mi_params.mi_cols;
+ const int is_cdef_flag_changed =
+ cdef_info->prev_cdef_enable_flag != cm->seq_params.enable_cdef;
+ const int is_large_scale_tile_changed =
+ cdef_info->prev_large_scale_tile_flag != cm->tiles.large_scale;
+ const int is_num_planes_changed = cdef_info->prev_num_planes != num_planes;
// num-bufs=2 represents ping-pong buffers for top linebuf.
// this is to avoid linebuf over-write by consecutive row.
int num_bufs = 2;
- if (is_frame_scaled) av1_free_cdef_linebuf(cm);
+ if (is_frame_scaled || is_sub_sampling_changed || is_cdef_flag_changed ||
+ is_large_scale_tile_changed || is_num_planes_changed)
+ av1_free_cdef_linebuf(cm);
+
+ // Store configuration to check change in configuration
+ cdef_info->allocated_mi_cols = cm->mi_params.mi_cols;
+ cdef_info->allocated_subsampling_x = cm->seq_params.subsampling_x;
+ cdef_info->allocated_subsampling_y = cm->seq_params.subsampling_y;
+ cdef_info->prev_cdef_enable_flag = cm->seq_params.enable_cdef;
+ cdef_info->prev_large_scale_tile_flag = cm->tiles.large_scale;
+ cdef_info->prev_num_planes = num_planes;
+
+ if (!cm->seq_params.enable_cdef && cm->tiles.large_scale) return;
for (int plane = 0; plane < num_planes; plane++) {
- if (cm->cdef_info.linebuf[plane] == NULL) {
+ if (cdef_info->linebuf[plane] == NULL) {
const int stride =
luma_stride >>
(plane == AOM_PLANE_Y ? 0 : cm->seq_params.subsampling_x);
- CHECK_MEM_ERROR(cm, cm->cdef_info.linebuf[plane],
- aom_malloc(sizeof(*cm->cdef_info.linebuf) * num_bufs *
+ CHECK_MEM_ERROR(cm, cdef_info->linebuf[plane],
+ aom_malloc(sizeof(*cdef_info->linebuf) * num_bufs *
(CDEF_VBORDER << 1) * stride));
}
}
- cm->cdef_info.allocated_mi_cols = cm->mi_params.mi_cols;
- cm->cdef_info.allocated_mi_rows = cm->mi_params.mi_rows;
}
#if !CONFIG_REALTIME_ONLY
diff --git a/av1/common/av1_common_int.h b/av1/common/av1_common_int.h
index ab82846..e8831c7 100644
--- a/av1/common/av1_common_int.h
+++ b/av1/common/av1_common_int.h
@@ -200,9 +200,13 @@
int cdef_strengths[CDEF_MAX_STRENGTHS]; /*!< CDEF strength values for luma */
int cdef_uv_strengths[CDEF_MAX_STRENGTHS]; /*!< CDEF strength values for
chroma */
- int cdef_bits; /*!< Number of CDEF strength values in bits */
- int allocated_mi_cols; /*!< Number of cols in the frame in 4 pixel */
- int allocated_mi_rows; /*!< Number of rows in the frame in 4 pixel */
+ int cdef_bits; /*!< Number of CDEF strength values in bits */
+ int allocated_mi_cols; /*!< Number of cols in the frame in 4 pixel */
+ int allocated_subsampling_x; /*!< Chroma subsampling for x */
+ int allocated_subsampling_y; /*!< Chroma subsampling for y */
+ uint8_t prev_cdef_enable_flag; /*!< CDEF on/off flag */
+ unsigned int prev_large_scale_tile_flag; /*!< Large scale tile on/off flag */
+ int prev_num_planes; /*!< Number of planes */
} CdefInfo;
/*!\cond */
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 327acbb..e3e90a6 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2098,8 +2098,7 @@
aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
"Failed to allocate frame buffer");
- const int use_cdef = cm->seq_params.enable_cdef && !cm->tiles.large_scale;
- if (!is_stat_generation_stage(cpi) && use_cdef) av1_alloc_cdef_linebuf(cm);
+ if (!is_stat_generation_stage(cpi)) av1_alloc_cdef_linebuf(cm);
#if !CONFIG_REALTIME_ONLY
const int use_restoration = cm->seq_params.enable_restoration &&
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h
index 6049e74..1d9b772 100644
--- a/av1/encoder/encoder_alloc.h
+++ b/av1/encoder/encoder_alloc.h
@@ -252,8 +252,8 @@
#if !CONFIG_REALTIME_ONLY
av1_free_restoration_buffers(cm);
#endif
- const int use_cdef = cm->seq_params.enable_cdef && !cm->tiles.large_scale;
- if (use_cdef) av1_free_cdef_linebuf(cm);
+
+ if (!is_stat_generation_stage(cpi)) av1_free_cdef_linebuf(cm);
aom_free_frame_buffer(&cpi->trial_frame_rst);
aom_free_frame_buffer(&cpi->scaled_source);
