commit 522e042413af28ca95955d13e085563589f9c0c1
author Satheesh Kumar <satheesh.kumar@ittiam.com> Mon Nov 06 10:59:12 2023 +0530
committer James Zern <jzern@google.com> Tue Nov 07 22:59:18 2023 +0000
tree a0279d74305da9eb5ff615668b2bf9f9c9d7c82f
parent 23f023cf3a07648e7f78de2777d88bcfef73548d

Add missing allocation checks in realloc_frame_buffer_aligned

This CL adds memory allocation checks for aom_alloc_pyramid() and
av1_alloc_corner_list() calls in realloc_frame_buffer_aligned().
Also corrects the freeing order of previously allocated buffers in
case of allocation failure in aom_alloc_pyramid().

Bug: aomedia:3276
Change-Id: Ifecdddc4fc82deb6913eb70011731df6997c78ee

aom_dsp/flow_estimation/corner_detect.c

diff --git a/aom_dsp/flow_estimation/corner_detect.c b/aom_dsp/flow_estimation/corner_detect.c
index 8a6f7cb..284d1bd 100644
--- a/aom_dsp/flow_estimation/corner_detect.c
+++ b/aom_dsp/flow_estimation/corner_detect.c
@@ -27,7 +27,7 @@
 size_t av1_get_corner_list_size(void) { return sizeof(CornerList); }
 
 CornerList *av1_alloc_corner_list(void) {
-  CornerList *corners = (CornerList *)aom_calloc(1, sizeof(CornerList));
+  CornerList *corners = (CornerList *)aom_calloc(1, sizeof(*corners));
   if (!corners) {
     return NULL;
   }

aom_dsp/pyramid.c

diff --git a/aom_dsp/pyramid.c b/aom_dsp/pyramid.c
index 11f9f1a..324a18b 100644
--- a/aom_dsp/pyramid.c
+++ b/aom_dsp/pyramid.c
@@ -112,7 +112,7 @@
     return NULL;
   }
 
-  pyr->layers = aom_calloc(n_levels, sizeof(PyramidLayer));
+  pyr->layers = aom_calloc(n_levels, sizeof(*pyr->layers));
   if (!pyr->layers) {
     aom_free(pyr);
     return NULL;
@@ -125,10 +125,10 @@
   // These are gathered up first, so that we can allocate all pyramid levels
   // in a single buffer
   size_t buffer_size = 0;
-  size_t *layer_offsets = aom_calloc(n_levels, sizeof(size_t));
+  size_t *layer_offsets = aom_calloc(n_levels, sizeof(*layer_offsets));
   if (!layer_offsets) {
-    aom_free(pyr);
     aom_free(pyr->layers);
+    aom_free(pyr);
     return NULL;
   }
 
@@ -195,8 +195,8 @@
   pyr->buffer_alloc =
       aom_memalign(PYRAMID_ALIGNMENT, buffer_size * sizeof(*pyr->buffer_alloc));
   if (!pyr->buffer_alloc) {
-    aom_free(pyr);
     aom_free(pyr->layers);
+    aom_free(pyr);
     aom_free(layer_offsets);
     return NULL;
   }

aom_scale/generic/yv12config.c

diff --git a/aom_scale/generic/yv12config.c b/aom_scale/generic/yv12config.c
index 82376f4..94b400b 100644
--- a/aom_scale/generic/yv12config.c
+++ b/aom_scale/generic/yv12config.c
@@ -193,7 +193,9 @@
     if (num_pyramid_levels > 0) {
       ybf->y_pyramid = aom_alloc_pyramid(width, height, num_pyramid_levels,
                                          use_highbitdepth);
+      if (!ybf->y_pyramid) return AOM_CODEC_MEM_ERROR;
       ybf->corners = av1_alloc_corner_list();
+      if (!ybf->corners) return AOM_CODEC_MEM_ERROR;
     }
 #endif  // CONFIG_AV1_ENCODER && !CONFIG_REALTIME_ONLY