commit 12adc723acf02633595a4d8da8345742729f46c0
author Wan-Teh Chang <wtc@google.com> Fri Apr 30 14:45:52 2021 -0700
committer Wan-Teh Chang <wtc@google.com> Sat May 01 00:32:00 2021 +0000
tree c34488c0d1841ce9c70ad15e0fedf54c87c9f338
parent 4efe20e99dcd9b6f8eadc8de8acc825be7416578

aom_dsp: Fix aom_noise_strength_lut_init with zero size

Return 0 when lut size is zero. aom_noise_strength_lut_eval() assumes
lut size is nonzero. For example, it accesses lut->points[0] and
lut->points[lut->num_points - 1] without checking lut->num_points.

Also handle the failure of aom_noise_strength_solver_fit_piecewise().

BUG=aomedia:2999

Change-Id: I19858c8bd097f523f6316effedb5e3b9acd2e4b7

aom_dsp/noise_model.c

diff --git a/aom_dsp/noise_model.c b/aom_dsp/noise_model.c
index 0b74009..19c660e 100644
--- a/aom_dsp/noise_model.c
+++ b/aom_dsp/noise_model.c
@@ -214,7 +214,7 @@
 
 int aom_noise_strength_lut_init(aom_noise_strength_lut_t *lut, int num_points) {
   if (!lut) return 0;
-  if (num_points < 0) return 0;
+  if (num_points <= 0) return 0;
   lut->num_points = 0;
   lut->points = (double(*)[2])aom_malloc(num_points * sizeof(*lut->points));
   if (!lut->points) return 0;
@@ -1153,12 +1153,24 @@
 
   // Convert the scaling functions to 8 bit values
   aom_noise_strength_lut_t scaling_points[3];
-  aom_noise_strength_solver_fit_piecewise(
-      &noise_model->combined_state[0].strength_solver, 14, scaling_points + 0);
-  aom_noise_strength_solver_fit_piecewise(
-      &noise_model->combined_state[1].strength_solver, 10, scaling_points + 1);
-  aom_noise_strength_solver_fit_piecewise(
-      &noise_model->combined_state[2].strength_solver, 10, scaling_points + 2);
+  if (!aom_noise_strength_solver_fit_piecewise(
+          &noise_model->combined_state[0].strength_solver, 14,
+          scaling_points + 0)) {
+    return 0;
+  }
+  if (!aom_noise_strength_solver_fit_piecewise(
+          &noise_model->combined_state[1].strength_solver, 10,
+          scaling_points + 1)) {
+    aom_noise_strength_lut_free(scaling_points + 0);
+    return 0;
+  }
+  if (!aom_noise_strength_solver_fit_piecewise(
+          &noise_model->combined_state[2].strength_solver, 10,
+          scaling_points + 2)) {
+    aom_noise_strength_lut_free(scaling_points + 0);
+    aom_noise_strength_lut_free(scaling_points + 1);
+    return 0;
+  }
 
   // Both the domain and the range of the scaling functions in the film_grain
   // are normalized to 8-bit (e.g., they are implicitly scaled during grain

test/noise_model_test.cc

diff --git a/test/noise_model_test.cc b/test/noise_model_test.cc
index 8584bd8..c12c080 100644
--- a/test/noise_model_test.cc
+++ b/test/noise_model_test.cc
@@ -212,9 +212,10 @@
   aom_noise_strength_solver_free(&solver);
 }
 
-TEST(NoiseStrengthLut, LutInitNegativeSize) {
+TEST(NoiseStrengthLut, LutInitNegativeOrZeroSize) {
   aom_noise_strength_lut_t lut;
   ASSERT_FALSE(aom_noise_strength_lut_init(&lut, -1));
+  ASSERT_FALSE(aom_noise_strength_lut_init(&lut, 0));
 }
 
 TEST(NoiseStrengthLut, LutEvalSinglePoint) {