temporal_filter: add missing alloc checks
in tf_alloc_and_reset_data() and av1_apply_temporal_filter_c();
tf_alloc_and_reset_data() now returns a bool
Bug: aomedia:3276
Change-Id: I290416607a21bc3decddbdf14b6dcc0eef337776
diff --git a/av1/encoder/ethread.c b/av1/encoder/ethread.c
index 3bd8c1f..8b9f93a 100644
--- a/av1/encoder/ethread.c
+++ b/av1/encoder/ethread.c
@@ -1995,8 +1995,11 @@
// OBMC buffers are used only to init MS params and remain unused when
// called from tf, hence set the buffers to defaults.
av1_init_obmc_buffer(&thread_data->td->mb.obmc_buffer);
- tf_alloc_and_reset_data(&thread_data->td->tf_data, cpi->tf_ctx.num_pels,
- is_highbitdepth);
+ if (!tf_alloc_and_reset_data(&thread_data->td->tf_data,
+ cpi->tf_ctx.num_pels, is_highbitdepth)) {
+ aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR,
+ "Error allocating temporal filter data");
+ }
}
}
}
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index dbdd66d..5f76b18 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -608,11 +608,20 @@
// Allocate memory for pixel-wise squared differences. They,
// regardless of the subsampling, are assigned with memory of size `mb_pels`.
uint32_t *square_diff = aom_memalign(16, mb_pels * sizeof(uint32_t));
+ if (!square_diff) {
+ aom_internal_error(mbd->error_info, AOM_CODEC_MEM_ERROR,
+ "Error allocating temporal filter data");
+ }
memset(square_diff, 0, mb_pels * sizeof(square_diff[0]));
// Allocate memory for accumulated luma squared error. This value will be
// consumed while filtering the chroma planes.
uint32_t *luma_sse_sum = aom_memalign(32, mb_pels * sizeof(uint32_t));
+ if (!luma_sse_sum) {
+ aom_free(square_diff);
+ aom_internal_error(mbd->error_info, AOM_CODEC_MEM_ERROR,
+ "Error allocating temporal filter data");
+ }
memset(luma_sse_sum, 0, mb_pels * sizeof(luma_sse_sum[0]));
// Get window size for pixel-wise filtering.
@@ -1245,7 +1254,10 @@
// Allocate and reset temporal filter buffers.
const int is_highbitdepth = tf_ctx->is_highbitdepth;
- tf_alloc_and_reset_data(tf_data, tf_ctx->num_pels, is_highbitdepth);
+ if (!tf_alloc_and_reset_data(tf_data, tf_ctx->num_pels, is_highbitdepth)) {
+ aom_internal_error(cpi->common.error, AOM_CODEC_MEM_ERROR,
+ "Error allocating temporal filter data");
+ }
// Perform temporal filtering process.
if (mt_info->num_workers > 1)
diff --git a/av1/encoder/temporal_filter.h b/av1/encoder/temporal_filter.h
index f6ae27f..8225dd9 100644
--- a/av1/encoder/temporal_filter.h
+++ b/av1/encoder/temporal_filter.h
@@ -12,6 +12,8 @@
#ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
#define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
+#include <stdbool.h>
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -348,7 +350,7 @@
// is_high_bitdepth: Whether the frame is high-bitdepth or not.
// Returns:
// Nothing will be returned. But the contents of tf_data will be modified.
-static AOM_INLINE void tf_alloc_and_reset_data(TemporalFilterData *tf_data,
+static AOM_INLINE bool tf_alloc_and_reset_data(TemporalFilterData *tf_data,
int num_pels,
int is_high_bitdepth) {
tf_data->tmp_mbmi = (MB_MODE_INFO *)malloc(sizeof(*tf_data->tmp_mbmi));
@@ -364,6 +366,13 @@
else
tf_data->pred =
(uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
+ if (!(tf_data->accum && tf_data->count && tf_data->pred)) {
+ aom_free(tf_data->accum);
+ aom_free(tf_data->count);
+ aom_free(tf_data->pred);
+ return false;
+ }
+ return true;
}
// Setup macroblockd params for temporal filtering process.
