resize.c: add missing alloc checks
Bug: aomedia:3276
Change-Id: Ib67c35bae2a403cc44599ce3f5f05ab3709a1fcb
diff --git a/av1/common/resize.c b/av1/common/resize.c
index 8591f98..fe9d1dc 100644
--- a/av1/common/resize.c
+++ b/av1/common/resize.c
@@ -12,6 +12,7 @@
#include <assert.h>
#include <limits.h>
#include <math.h>
+#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -700,7 +701,7 @@
aom_free(arrbuf2);
}
-static void upscale_normative_rect(const uint8_t *const input, int height,
+static bool upscale_normative_rect(const uint8_t *const input, int height,
int width, int in_stride, uint8_t *output,
int height2, int width2, int out_stride,
int x_step_qn, int x0_qn, int pad_left,
@@ -725,6 +726,7 @@
uint8_t *const in_tr = (uint8_t *)(input + width);
if (pad_left) {
tmp_left = (uint8_t *)aom_malloc(sizeof(*tmp_left) * border_cols * height);
+ if (!tmp_left) return false;
for (int i = 0; i < height; i++) {
memcpy(tmp_left + i * border_cols, in_tl + i * in_stride, border_cols);
memset(in_tl + i * in_stride, input[i * in_stride], border_cols);
@@ -733,6 +735,10 @@
if (pad_right) {
tmp_right =
(uint8_t *)aom_malloc(sizeof(*tmp_right) * border_cols * height);
+ if (!tmp_right) {
+ aom_free(tmp_left);
+ return false;
+ }
for (int i = 0; i < height; i++) {
memcpy(tmp_right + i * border_cols, in_tr + i * in_stride, border_cols);
memset(in_tr + i * in_stride, input[i * in_stride + width - 1],
@@ -757,6 +763,7 @@
}
aom_free(tmp_right);
}
+ return true;
}
#if CONFIG_AV1_HIGHBITDEPTH
@@ -1045,7 +1052,7 @@
aom_free(arrbuf2);
}
-static void highbd_upscale_normative_rect(const uint8_t *const input,
+static bool highbd_upscale_normative_rect(const uint8_t *const input,
int height, int width, int in_stride,
uint8_t *output, int height2,
int width2, int out_stride,
@@ -1073,6 +1080,7 @@
uint16_t *const in_tr = input16 + width;
if (pad_left) {
tmp_left = (uint16_t *)aom_malloc(sizeof(*tmp_left) * border_cols * height);
+ if (!tmp_left) return false;
for (int i = 0; i < height; i++) {
memcpy(tmp_left + i * border_cols, in_tl + i * in_stride, border_size);
aom_memset16(in_tl + i * in_stride, input16[i * in_stride], border_cols);
@@ -1081,6 +1089,10 @@
if (pad_right) {
tmp_right =
(uint16_t *)aom_malloc(sizeof(*tmp_right) * border_cols * height);
+ if (!tmp_right) {
+ aom_free(tmp_left);
+ return false;
+ }
for (int i = 0; i < height; i++) {
memcpy(tmp_right + i * border_cols, in_tr + i * in_stride, border_size);
aom_memset16(in_tr + i * in_stride, input16[i * in_stride + width - 1],
@@ -1106,6 +1118,7 @@
}
aom_free(tmp_right);
}
+ return true;
}
#endif // CONFIG_AV1_HIGHBITDEPTH
@@ -1304,21 +1317,26 @@
const int pad_left = (j == 0);
const int pad_right = (j == cm->tiles.cols - 1);
+ bool success;
#if CONFIG_AV1_HIGHBITDEPTH
if (cm->seq_params->use_highbitdepth)
- highbd_upscale_normative_rect(src_ptr, rows, src_width, src_stride,
- dst_ptr, rows, dst_width, dst_stride,
- x_step_qn, x0_qn, pad_left, pad_right,
- cm->seq_params->bit_depth);
+ success = highbd_upscale_normative_rect(
+ src_ptr, rows, src_width, src_stride, dst_ptr, rows, dst_width,
+ dst_stride, x_step_qn, x0_qn, pad_left, pad_right,
+ cm->seq_params->bit_depth);
else
- upscale_normative_rect(src_ptr, rows, src_width, src_stride, dst_ptr,
- rows, dst_width, dst_stride, x_step_qn, x0_qn,
- pad_left, pad_right);
+ success = upscale_normative_rect(src_ptr, rows, src_width, src_stride,
+ dst_ptr, rows, dst_width, dst_stride,
+ x_step_qn, x0_qn, pad_left, pad_right);
#else
- upscale_normative_rect(src_ptr, rows, src_width, src_stride, dst_ptr, rows,
- dst_width, dst_stride, x_step_qn, x0_qn, pad_left,
- pad_right);
+ success = upscale_normative_rect(src_ptr, rows, src_width, src_stride,
+ dst_ptr, rows, dst_width, dst_stride,
+ x_step_qn, x0_qn, pad_left, pad_right);
#endif
+ if (!success) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Error upscaling frame");
+ }
// Update the fractional pixel offset to prepare for the next tile column.
x0_qn += (dst_width * x_step_qn) - (src_width << RS_SCALE_SUBPEL_BITS);
}
