commit 4dca33a3f7455e95d5449d9f5d01cc594ab27eeb
author James Zern <jzern@google.com> Thu Apr 28 12:54:06 2022 -0700
committer James Zern <jzern@google.com> Wed May 11 20:45:53 2022 +0000
tree ef4165a02cc50efdf9d2df9e6aae73d0300603dd
parent 014e146ce0f9f251a6ec4d01e033e79422811e11

cdef_alloc_data: add missing alloc checks

Bug: aomedia:3276
Change-Id: Ida6a9efb489c8a7bca141b881265f5deccaf906f

av1/encoder/pickcdef.c

diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c
index 7f7d0c9..557c9ee 100644
--- a/av1/encoder/pickcdef.c
+++ b/av1/encoder/pickcdef.c
@@ -10,6 +10,7 @@
  */
 
 #include <math.h>
+#include <stdbool.h>
 #include <string.h>
 
 #include "config/aom_dsp_rtcd.h"
@@ -413,7 +414,7 @@
 //   related to CDEF search context.
 // Returns:
 //   Nothing will be returned. Contents of cdef_search_ctx will be modified.
-static AOM_INLINE void cdef_alloc_data(CdefSearchCtx *cdef_search_ctx) {
+static AOM_INLINE bool cdef_alloc_data(CdefSearchCtx *cdef_search_ctx) {
   const int nvfb = cdef_search_ctx->nvfb;
   const int nhfb = cdef_search_ctx->nhfb;
   cdef_search_ctx->sb_index =
@@ -423,6 +424,14 @@
       aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb);
   cdef_search_ctx->mse[1] =
       aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb);
+  if (!(cdef_search_ctx->sb_index && cdef_search_ctx->mse[0] &&
+        cdef_search_ctx->mse[1])) {
+    aom_free(cdef_search_ctx->sb_index);
+    aom_free(cdef_search_ctx->mse[0]);
+    aom_free(cdef_search_ctx->mse[1]);
+    return false;
+  }
+  return true;
 }
 
 // Deallocates the memory allocated for members of CdefSearchCtx.
@@ -599,7 +608,14 @@
   // Initialize parameters related to CDEF search context.
   cdef_params_init(frame, ref, cm, xd, &cdef_search_ctx, pick_method);
   // Allocate CDEF search context buffers.
-  cdef_alloc_data(&cdef_search_ctx);
+  if (!cdef_alloc_data(&cdef_search_ctx)) {
+    CdefInfo *const cdef_info = &cm->cdef_info;
+    cdef_info->nb_cdef_strengths = 0;
+    cdef_info->cdef_bits = 0;
+    cdef_info->cdef_strengths[0] = 0;
+    cdef_info->cdef_uv_strengths[0] = 0;
+    return;
+  }
   // Frame level mse calculation.
   if (mt_info->num_workers > 1) {
     av1_cdef_mse_calc_frame_mt(cm, mt_info, &cdef_search_ctx);