Handle allocation failure in smooth_filter_noise()

This CL handles the allocation failure of 'smooth_noise' buffer in
'smooth_filter_noise()' by calling 'aom_internal_error()' in the
caller function using 'error_info' derived from 'cpi->common'.

Bug: aomedia:3276
Change-Id: Id72a406f0fbedd307c44c0835532922e1351509e

av1/encoder/pass2_strategy.c

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 3641c8b..cb9a8c8 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -3536,12 +3536,13 @@
 }
 
 // Smooth-out the noise variance so it is more stable
+// Returns 0 on success, -1 on memory allocation failure.
 // TODO(bohanli): Use a better low-pass filter than averaging
-static void smooth_filter_noise(FIRSTPASS_STATS *first_stats,
-                                FIRSTPASS_STATS *last_stats) {
+static int smooth_filter_noise(FIRSTPASS_STATS *first_stats,
+                               FIRSTPASS_STATS *last_stats) {
   int len = (int)(last_stats - first_stats);
   double *smooth_noise = aom_malloc(len * sizeof(*smooth_noise));
-  if (!smooth_noise) return;
+  if (!smooth_noise) return -1;
 
   for (int i = 0; i < len; i++) {
     double total_noise = 0;
@@ -3566,11 +3567,13 @@
   }
 
   aom_free(smooth_noise);
+  return 0;
 }
 
 // Estimate the noise variance of each frame from the first pass stats
 void av1_estimate_noise(FIRSTPASS_STATS *first_stats,
-                        FIRSTPASS_STATS *last_stats) {
+                        FIRSTPASS_STATS *last_stats,
+                        struct aom_internal_error_info *error_info) {
   FIRSTPASS_STATS *this_stats, *next_stats;
   double C1, C2, C3, noise;
   for (this_stats = first_stats + 2; this_stats < last_stats; this_stats++) {
@@ -3656,7 +3659,10 @@
     this_stats->noise_var = (first_stats + 2)->noise_var;
   }
 
-  smooth_filter_noise(first_stats, last_stats);
+  if (smooth_filter_noise(first_stats, last_stats) == -1) {
+    aom_internal_error(error_info, AOM_CODEC_MEM_ERROR,
+                       "Error allocating buffers in smooth_filter_noise()");
+  }
 }
 
 // Estimate correlation coefficient of each frame with its previous frame.
@@ -3823,7 +3829,8 @@
         av1_mark_flashes(twopass->stats_buf_ctx->stats_in_start,
                          twopass->stats_buf_ctx->stats_in_end);
         av1_estimate_noise(twopass->stats_buf_ctx->stats_in_start,
-                           twopass->stats_buf_ctx->stats_in_end);
+                           twopass->stats_buf_ctx->stats_in_end,
+                           cpi->common.error);
         av1_estimate_coeff(twopass->stats_buf_ctx->stats_in_start,
                            twopass->stats_buf_ctx->stats_in_end);
         ret = identify_regions(cpi->twopass_frame.stats_in, rest_frames,
@@ -3997,7 +4004,7 @@
   av1_mark_flashes(twopass->stats_buf_ctx->stats_in_start,
                    twopass->stats_buf_ctx->stats_in_end);
   av1_estimate_noise(twopass->stats_buf_ctx->stats_in_start,
-                     twopass->stats_buf_ctx->stats_in_end);
+                     twopass->stats_buf_ctx->stats_in_end, cpi->common.error);
   av1_estimate_coeff(twopass->stats_buf_ctx->stats_in_start,
                      twopass->stats_buf_ctx->stats_in_end);
 

av1/encoder/pass2_strategy.h

diff --git a/av1/encoder/pass2_strategy.h b/av1/encoder/pass2_strategy.h
index ff1591c..5987a78 100644
--- a/av1/encoder/pass2_strategy.h
+++ b/av1/encoder/pass2_strategy.h
@@ -137,7 +137,8 @@
 void av1_mark_flashes(FIRSTPASS_STATS *first_stats,
                       FIRSTPASS_STATS *last_stats);
 void av1_estimate_noise(FIRSTPASS_STATS *first_stats,
-                        FIRSTPASS_STATS *last_stats);
+                        FIRSTPASS_STATS *last_stats,
+                        struct aom_internal_error_info *error_info);
 void av1_estimate_coeff(FIRSTPASS_STATS *first_stats,
                         FIRSTPASS_STATS *last_stats);