grain_synthesis.c: add missing alloc checks
Bug: aomedia:3276
Change-Id: I06c0f501951adeda5255032d3caff68eb2bfab4e
diff --git a/av1/decoder/grain_synthesis.c b/av1/decoder/grain_synthesis.c
index 60b458b..5927eba 100644
--- a/av1/decoder/grain_synthesis.c
+++ b/av1/decoder/grain_synthesis.c
@@ -14,6 +14,7 @@
*
*/
+#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
@@ -238,7 +239,61 @@
static uint16_t random_register = 0; // random number generator register
-static void init_arrays(const aom_film_grain_t *params, int luma_stride,
+static void dealloc_arrays(const aom_film_grain_t *params, int ***pred_pos_luma,
+ int ***pred_pos_chroma, int **luma_grain_block,
+ int **cb_grain_block, int **cr_grain_block,
+ int **y_line_buf, int **cb_line_buf,
+ int **cr_line_buf, int **y_col_buf, int **cb_col_buf,
+ int **cr_col_buf) {
+ int num_pos_luma = 2 * params->ar_coeff_lag * (params->ar_coeff_lag + 1);
+ int num_pos_chroma = num_pos_luma;
+ if (params->num_y_points > 0) ++num_pos_chroma;
+
+ if (*pred_pos_luma) {
+ for (int row = 0; row < num_pos_luma; row++) {
+ aom_free((*pred_pos_luma)[row]);
+ }
+ aom_free(*pred_pos_luma);
+ *pred_pos_luma = NULL;
+ }
+
+ if (*pred_pos_chroma) {
+ for (int row = 0; row < num_pos_chroma; row++) {
+ aom_free((*pred_pos_chroma)[row]);
+ }
+ aom_free(*pred_pos_chroma);
+ *pred_pos_chroma = NULL;
+ }
+
+ aom_free(*y_line_buf);
+ *y_line_buf = NULL;
+
+ aom_free(*cb_line_buf);
+ *cb_line_buf = NULL;
+
+ aom_free(*cr_line_buf);
+ *cr_line_buf = NULL;
+
+ aom_free(*y_col_buf);
+ *y_col_buf = NULL;
+
+ aom_free(*cb_col_buf);
+ *cb_col_buf = NULL;
+
+ aom_free(*cr_col_buf);
+ *cr_col_buf = NULL;
+
+ aom_free(*luma_grain_block);
+ *luma_grain_block = NULL;
+
+ aom_free(*cb_grain_block);
+ *cb_grain_block = NULL;
+
+ aom_free(*cr_grain_block);
+ *cr_grain_block = NULL;
+}
+
+static bool init_arrays(const aom_film_grain_t *params, int luma_stride,
int chroma_stride, int ***pred_pos_luma_p,
int ***pred_pos_chroma_p, int **luma_grain_block,
int **cb_grain_block, int **cr_grain_block,
@@ -246,6 +301,18 @@
int **y_col_buf, int **cb_col_buf, int **cr_col_buf,
int luma_grain_samples, int chroma_grain_samples,
int chroma_subsamp_y, int chroma_subsamp_x) {
+ *pred_pos_luma_p = NULL;
+ *pred_pos_chroma_p = NULL;
+ *luma_grain_block = NULL;
+ *cb_grain_block = NULL;
+ *cr_grain_block = NULL;
+ *y_line_buf = NULL;
+ *cb_line_buf = NULL;
+ *cr_line_buf = NULL;
+ *y_col_buf = NULL;
+ *cb_col_buf = NULL;
+ *cr_col_buf = NULL;
+
memset(scaling_lut_y, 0, sizeof(*scaling_lut_y) * 256);
memset(scaling_lut_cb, 0, sizeof(*scaling_lut_cb) * 256);
memset(scaling_lut_cr, 0, sizeof(*scaling_lut_cr) * 256);
@@ -257,17 +324,38 @@
int **pred_pos_luma;
int **pred_pos_chroma;
- pred_pos_luma = (int **)aom_malloc(sizeof(*pred_pos_luma) * num_pos_luma);
+ pred_pos_luma = (int **)aom_calloc(num_pos_luma, sizeof(*pred_pos_luma));
+ if (!pred_pos_luma) return false;
for (int row = 0; row < num_pos_luma; row++) {
pred_pos_luma[row] = (int *)aom_malloc(sizeof(**pred_pos_luma) * 3);
+ if (!pred_pos_luma[row]) {
+ dealloc_arrays(params, pred_pos_luma_p, pred_pos_chroma_p,
+ luma_grain_block, cb_grain_block, cr_grain_block,
+ y_line_buf, cb_line_buf, cr_line_buf, y_col_buf,
+ cb_col_buf, cr_col_buf);
+ return false;
+ }
}
pred_pos_chroma =
- (int **)aom_malloc(sizeof(*pred_pos_chroma) * num_pos_chroma);
+ (int **)aom_calloc(num_pos_chroma, sizeof(*pred_pos_chroma));
+ if (!pred_pos_chroma) {
+ dealloc_arrays(params, pred_pos_luma_p, pred_pos_chroma_p, luma_grain_block,
+ cb_grain_block, cr_grain_block, y_line_buf, cb_line_buf,
+ cr_line_buf, y_col_buf, cb_col_buf, cr_col_buf);
+ return false;
+ }
for (int row = 0; row < num_pos_chroma; row++) {
pred_pos_chroma[row] = (int *)aom_malloc(sizeof(**pred_pos_chroma) * 3);
+ if (!pred_pos_chroma[row]) {
+ dealloc_arrays(params, pred_pos_luma_p, pred_pos_chroma_p,
+ luma_grain_block, cb_grain_block, cr_grain_block,
+ y_line_buf, cb_line_buf, cr_line_buf, y_col_buf,
+ cb_col_buf, cr_col_buf);
+ return false;
+ }
}
int pos_ar_index = 0;
@@ -330,45 +418,15 @@
(int *)aom_malloc(sizeof(**cb_grain_block) * chroma_grain_samples);
*cr_grain_block =
(int *)aom_malloc(sizeof(**cr_grain_block) * chroma_grain_samples);
-}
-
-static void dealloc_arrays(const aom_film_grain_t *params, int ***pred_pos_luma,
- int ***pred_pos_chroma, int **luma_grain_block,
- int **cb_grain_block, int **cr_grain_block,
- int **y_line_buf, int **cb_line_buf,
- int **cr_line_buf, int **y_col_buf, int **cb_col_buf,
- int **cr_col_buf) {
- int num_pos_luma = 2 * params->ar_coeff_lag * (params->ar_coeff_lag + 1);
- int num_pos_chroma = num_pos_luma;
- if (params->num_y_points > 0) ++num_pos_chroma;
-
- for (int row = 0; row < num_pos_luma; row++) {
- aom_free((*pred_pos_luma)[row]);
+ if (!(*pred_pos_luma_p && *pred_pos_chroma_p && *y_line_buf && *cb_line_buf &&
+ *cr_line_buf && *y_col_buf && *cb_col_buf && *cr_col_buf &&
+ *luma_grain_block && *cb_grain_block && *cr_grain_block)) {
+ dealloc_arrays(params, pred_pos_luma_p, pred_pos_chroma_p, luma_grain_block,
+ cb_grain_block, cr_grain_block, y_line_buf, cb_line_buf,
+ cr_line_buf, y_col_buf, cb_col_buf, cr_col_buf);
+ return false;
}
- aom_free(*pred_pos_luma);
-
- for (int row = 0; row < num_pos_chroma; row++) {
- aom_free((*pred_pos_chroma)[row]);
- }
- aom_free((*pred_pos_chroma));
-
- aom_free(*y_line_buf);
-
- aom_free(*cb_line_buf);
-
- aom_free(*cr_line_buf);
-
- aom_free(*y_col_buf);
-
- aom_free(*cb_col_buf);
-
- aom_free(*cr_col_buf);
-
- aom_free(*luma_grain_block);
-
- aom_free(*cb_grain_block);
-
- aom_free(*cr_grain_block);
+ return true;
}
// get a number between 0 and 2^bits - 1
@@ -1081,13 +1139,14 @@
grain_min = 0 - grain_center;
grain_max = grain_center - 1;
- init_arrays(params, luma_stride, chroma_stride, &pred_pos_luma,
- &pred_pos_chroma, &luma_grain_block, &cb_grain_block,
- &cr_grain_block, &y_line_buf, &cb_line_buf, &cr_line_buf,
- &y_col_buf, &cb_col_buf, &cr_col_buf,
- luma_block_size_y * luma_block_size_x,
- chroma_block_size_y * chroma_block_size_x, chroma_subsamp_y,
- chroma_subsamp_x);
+ if (!init_arrays(params, luma_stride, chroma_stride, &pred_pos_luma,
+ &pred_pos_chroma, &luma_grain_block, &cb_grain_block,
+ &cr_grain_block, &y_line_buf, &cb_line_buf, &cr_line_buf,
+ &y_col_buf, &cb_col_buf, &cr_col_buf,
+ luma_block_size_y * luma_block_size_x,
+ chroma_block_size_y * chroma_block_size_x, chroma_subsamp_y,
+ chroma_subsamp_x))
+ return -1;
if (generate_luma_grain_block(params, pred_pos_luma, luma_grain_block,
luma_block_size_y, luma_block_size_x,
