commit a26f94609c90f4e6dfe5461b968d6326c23f5caa
author James Zern <jzern@google.com> Mon Feb 28 19:46:31 2022 -0800
committer James Zern <jzern@google.com> Tue Mar 01 19:29:40 2022 +0000
tree fe876718723da60c9839ae25e832465664f4b13b
parent 2a7be8425fdf56158cf99eb78f912bbadc7edb1d

aom_noise_model_init: add additional error checking

check coords allocation and that the bitdepth is within the expected
range

Bug: aom:3190
Change-Id: I98ad520bcc2eab6ce63d0b1f654b049c7e7897e3

aom_dsp/noise_model.c

diff --git a/aom_dsp/noise_model.c b/aom_dsp/noise_model.c
index fd43b62..61b10fb 100644
--- a/aom_dsp/noise_model.c
+++ b/aom_dsp/noise_model.c
@@ -694,6 +694,10 @@
             kMaxLag);
     return 0;
   }
+  if (!(params.bit_depth == 8 || params.bit_depth == 10 ||
+        params.bit_depth == 12)) {
+    return 0;
+  }
 
   memcpy(&model->params, &params, sizeof(params));
   for (c = 0; c < 3; ++c) {
@@ -710,6 +714,10 @@
   }
   model->n = n;
   model->coords = (int(*)[2])aom_malloc(sizeof(*model->coords) * n);
+  if (!model->coords) {
+    aom_noise_model_free(model);
+    return 0;
+  }
 
   for (y = -lag; y <= 0; ++y) {
     const int max_x = y == 0 ? -1 : lag;