commit 9a0dd4bb2c743a9873a67c8377a1a60a39528406
author James Zern <jzern@google.com> Tue May 17 18:20:49 2022 -0700
committer James Zern <jzern@google.com> Sat May 21 02:27:21 2022 +0000
tree f0d8ab44434fb60b3306b51979724430198702a8
parent 9be1c519611ed152c21d3f23cb1741655ebeac99

ransac: fix alloc checks

check motions and current_motion.inlier_indices before attempting to
clear them

Bug: aomedia:3276
Change-Id: I9a876b9d5fa0037d8ff115295595de6a038d9664

av1/encoder/ransac.c

diff --git a/av1/encoder/ransac.c b/av1/encoder/ransac.c
index e849b34..f878fce 100644
--- a/av1/encoder/ransac.c
+++ b/av1/encoder/ransac.c
@@ -412,26 +412,29 @@
   corners1 = (double *)aom_malloc(sizeof(*corners1) * npoints * 2);
   corners2 = (double *)aom_malloc(sizeof(*corners2) * npoints * 2);
   image1_coord = (double *)aom_malloc(sizeof(*image1_coord) * npoints * 2);
-
   motions =
-      (RANSAC_MOTION *)aom_malloc(sizeof(RANSAC_MOTION) * num_desired_motions);
-  for (i = 0; i < num_desired_motions; ++i) {
-    motions[i].inlier_indices =
-        (int *)aom_malloc(sizeof(*motions->inlier_indices) * npoints);
-    clear_motion(motions + i, npoints);
-  }
+      (RANSAC_MOTION *)aom_calloc(num_desired_motions, sizeof(RANSAC_MOTION));
   current_motion.inlier_indices =
       (int *)aom_malloc(sizeof(*current_motion.inlier_indices) * npoints);
-  clear_motion(&current_motion, npoints);
-
-  worst_kept_motion = motions;
-
   if (!(points1 && points2 && corners1 && corners2 && image1_coord && motions &&
         current_motion.inlier_indices)) {
     ret_val = 1;
     goto finish_ransac;
   }
 
+  for (i = 0; i < num_desired_motions; ++i) {
+    motions[i].inlier_indices =
+        (int *)aom_malloc(sizeof(*motions->inlier_indices) * npoints);
+    if (!motions[i].inlier_indices) {
+      ret_val = 1;
+      goto finish_ransac;
+    }
+    clear_motion(motions + i, npoints);
+  }
+  clear_motion(&current_motion, npoints);
+
+  worst_kept_motion = motions;
+
   cnp1 = corners1;
   cnp2 = corners2;
   for (i = 0; i < npoints; ++i) {
@@ -542,10 +545,12 @@
   aom_free(corners2);
   aom_free(image1_coord);
   aom_free(current_motion.inlier_indices);
-  for (i = 0; i < num_desired_motions; ++i) {
-    aom_free(motions[i].inlier_indices);
+  if (motions) {
+    for (i = 0; i < num_desired_motions; ++i) {
+      aom_free(motions[i].inlier_indices);
+    }
+    aom_free(motions);
   }
-  aom_free(motions);
 
   return ret_val;
 }
@@ -593,26 +598,29 @@
   corners1 = (double *)aom_malloc(sizeof(*corners1) * npoints * 2);
   corners2 = (double *)aom_malloc(sizeof(*corners2) * npoints * 2);
   image1_coord = (double *)aom_malloc(sizeof(*image1_coord) * npoints * 2);
-
   motions =
-      (RANSAC_MOTION *)aom_malloc(sizeof(RANSAC_MOTION) * num_desired_motions);
-  for (i = 0; i < num_desired_motions; ++i) {
-    motions[i].inlier_indices =
-        (int *)aom_malloc(sizeof(*motions->inlier_indices) * npoints);
-    clear_motion(motions + i, npoints);
-  }
+      (RANSAC_MOTION *)aom_calloc(num_desired_motions, sizeof(RANSAC_MOTION));
   current_motion.inlier_indices =
       (int *)aom_malloc(sizeof(*current_motion.inlier_indices) * npoints);
-  clear_motion(&current_motion, npoints);
-
-  worst_kept_motion = motions;
-
   if (!(points1 && points2 && corners1 && corners2 && image1_coord && motions &&
         current_motion.inlier_indices)) {
     ret_val = 1;
     goto finish_ransac;
   }
 
+  for (i = 0; i < num_desired_motions; ++i) {
+    motions[i].inlier_indices =
+        (int *)aom_malloc(sizeof(*motions->inlier_indices) * npoints);
+    if (!motions[i].inlier_indices) {
+      ret_val = 1;
+      goto finish_ransac;
+    }
+    clear_motion(motions + i, npoints);
+  }
+  clear_motion(&current_motion, npoints);
+
+  worst_kept_motion = motions;
+
   cnp1 = corners1;
   cnp2 = corners2;
   for (i = 0; i < npoints; ++i) {
@@ -721,10 +729,12 @@
   aom_free(corners2);
   aom_free(image1_coord);
   aom_free(current_motion.inlier_indices);
-  for (i = 0; i < num_desired_motions; ++i) {
-    aom_free(motions[i].inlier_indices);
+  if (motions) {
+    for (i = 0; i < num_desired_motions; ++i) {
+      aom_free(motions[i].inlier_indices);
+    }
+    aom_free(motions);
   }
-  aom_free(motions);
 
   return ret_val;
 }