commit 150f3719c52e89490afaef2640be74cd0e7865c5
author Satheesh Kumar <satheesh.kumar@ittiam.com> Thu Oct 19 15:49:22 2023 +0530
committer James Zern <jzern@google.com> Wed Oct 25 19:50:58 2023 +0000
tree f60b1b93cc32e2e9bd4b84940dfb1a61047542bc
parent fe82063816d66850d7712e964e3001c8b35cf58c

Address allocation failure in tf_alloc_and_reset_data()

This CL addresses allocation failures for buffers allocated in
'tf_alloc_and_reset_data()'. Also replaces 'malloc()' and 'free()'
with 'aom_malloc()' and 'aom_free()' for the allocation and
deallocation of 'tf_data->tmp_mbmi' buffer.

Bug: aomedia:3276
Change-Id: I24460c4e7daae15703ed4f8a7e31247d1cf16271

av1/encoder/temporal_filter.h

diff --git a/av1/encoder/temporal_filter.h b/av1/encoder/temporal_filter.h
index d3e1762..2b15d28 100644
--- a/av1/encoder/temporal_filter.h
+++ b/av1/encoder/temporal_filter.h
@@ -356,29 +356,27 @@
 //   num_pels: Number of pixels in the block across all planes.
 //   is_high_bitdepth: Whether the frame is high-bitdepth or not.
 // Returns:
-//   Nothing will be returned. But the contents of tf_data will be modified.
+//   True if allocation is successful and false otherwise.
 static AOM_INLINE bool tf_alloc_and_reset_data(TemporalFilterData *tf_data,
                                                int num_pels,
                                                int is_high_bitdepth) {
-  tf_data->tmp_mbmi = (MB_MODE_INFO *)malloc(sizeof(*tf_data->tmp_mbmi));
-  memset(tf_data->tmp_mbmi, 0, sizeof(*tf_data->tmp_mbmi));
+  tf_data->tmp_mbmi = (MB_MODE_INFO *)aom_malloc(sizeof(*tf_data->tmp_mbmi));
   tf_data->accum =
       (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
   tf_data->count =
       (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
-  memset(&tf_data->diff, 0, sizeof(tf_data->diff));
   if (is_high_bitdepth)
     tf_data->pred = CONVERT_TO_BYTEPTR(
         aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
   else
     tf_data->pred =
         (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
-  if (!(tf_data->accum && tf_data->count && tf_data->pred)) {
-    aom_free(tf_data->accum);
-    aom_free(tf_data->count);
-    aom_free(tf_data->pred);
+  // In case of an allocation failure, other successfully allocated buffers will
+  // be freed by the tf_dealloc_data() call in encoder_destroy().
+  if (!(tf_data->tmp_mbmi && tf_data->accum && tf_data->count && tf_data->pred))
     return false;
-  }
+  memset(tf_data->tmp_mbmi, 0, sizeof(*tf_data->tmp_mbmi));
+  memset(&tf_data->diff, 0, sizeof(tf_data->diff));
   return true;
 }
 
@@ -408,7 +406,7 @@
                                        int is_high_bitdepth) {
   if (is_high_bitdepth)
     tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
-  free(tf_data->tmp_mbmi);
+  aom_free(tf_data->tmp_mbmi);
   tf_data->tmp_mbmi = NULL;
   aom_free(tf_data->accum);
   tf_data->accum = NULL;