aomdec.c: Verify we can scale a frame

Before scaling a frame, verify we can scale it, i.e., the frame's format
must be 4:2:0 and the frame's bit depth must be the same as the first
frame's bit depth.

Move the image_format_to_string() function from aomenc.c to
common/tools_common.c so that it can be called by aomdec.c.

Bug: aomedia:3424
Bug: aomedia:3425
Change-Id: I0b0348e902cb85e6bb1db5add8db1db27de22aa6
diff --git a/apps/aomdec.c b/apps/aomdec.c
index ab4a37f..333478a 100644
--- a/apps/aomdec.c
+++ b/apps/aomdec.c
@@ -9,7 +9,6 @@
  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
  */
 
-#include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
@@ -118,10 +117,18 @@
 };
 
 #if CONFIG_LIBYUV
-static INLINE int libyuv_scale(aom_image_t *src, aom_image_t *dst,
+// Returns 0 on success and returns -1 on failure.
+static INLINE int libyuv_scale(const aom_image_t *src, aom_image_t *dst,
                                FilterModeEnum mode) {
+  if (src->fmt != dst->fmt) {
+    fprintf(stderr,
+            "%s failed to scale output frame because format changed from %s to "
+            "%s\n",
+            exec_name, image_format_to_string(dst->fmt),
+            image_format_to_string(src->fmt));
+    return -1;
+  }
   if (src->fmt == AOM_IMG_FMT_I42016) {
-    assert(dst->fmt == AOM_IMG_FMT_I42016);
     return I420Scale_16(
         (uint16_t *)src->planes[AOM_PLANE_Y], src->stride[AOM_PLANE_Y] / 2,
         (uint16_t *)src->planes[AOM_PLANE_U], src->stride[AOM_PLANE_U] / 2,
@@ -131,15 +138,18 @@
         dst->stride[AOM_PLANE_U] / 2, (uint16_t *)dst->planes[AOM_PLANE_V],
         dst->stride[AOM_PLANE_V] / 2, dst->d_w, dst->d_h, mode);
   }
-  assert(src->fmt == AOM_IMG_FMT_I420);
-  assert(dst->fmt == AOM_IMG_FMT_I420);
-  return I420Scale(src->planes[AOM_PLANE_Y], src->stride[AOM_PLANE_Y],
-                   src->planes[AOM_PLANE_U], src->stride[AOM_PLANE_U],
-                   src->planes[AOM_PLANE_V], src->stride[AOM_PLANE_V], src->d_w,
-                   src->d_h, dst->planes[AOM_PLANE_Y], dst->stride[AOM_PLANE_Y],
-                   dst->planes[AOM_PLANE_U], dst->stride[AOM_PLANE_U],
-                   dst->planes[AOM_PLANE_V], dst->stride[AOM_PLANE_V], dst->d_w,
-                   dst->d_h, mode);
+  if (src->fmt == AOM_IMG_FMT_I420) {
+    return I420Scale(src->planes[AOM_PLANE_Y], src->stride[AOM_PLANE_Y],
+                     src->planes[AOM_PLANE_U], src->stride[AOM_PLANE_U],
+                     src->planes[AOM_PLANE_V], src->stride[AOM_PLANE_V],
+                     src->d_w, src->d_h, dst->planes[AOM_PLANE_Y],
+                     dst->stride[AOM_PLANE_Y], dst->planes[AOM_PLANE_U],
+                     dst->stride[AOM_PLANE_U], dst->planes[AOM_PLANE_V],
+                     dst->stride[AOM_PLANE_V], dst->d_w, dst->d_h, mode);
+  }
+  fprintf(stderr, "%s cannot scale output frame of format %s\n", exec_name,
+          image_format_to_string(src->fmt));
+  return -1;
 }
 #endif
 
@@ -878,7 +888,7 @@
 
           if (img->d_w != scaled_img->d_w || img->d_h != scaled_img->d_h) {
 #if CONFIG_LIBYUV
-            libyuv_scale(img, scaled_img, kFilterBox);
+            if (libyuv_scale(img, scaled_img, kFilterBox) != 0) goto fail;
             img = scaled_img;
 #else
             fprintf(
diff --git a/apps/aomenc.c b/apps/aomenc.c
index dcab898..d8f133d 100644
--- a/apps/aomenc.c
+++ b/apps/aomenc.c
@@ -1307,21 +1307,6 @@
   }
 }
 
-static const char *image_format_to_string(aom_img_fmt_t f) {
-  switch (f) {
-    case AOM_IMG_FMT_I420: return "I420";
-    case AOM_IMG_FMT_I422: return "I422";
-    case AOM_IMG_FMT_I444: return "I444";
-    case AOM_IMG_FMT_YV12: return "YV12";
-    case AOM_IMG_FMT_NV12: return "NV12";
-    case AOM_IMG_FMT_YV1216: return "YV1216";
-    case AOM_IMG_FMT_I42016: return "I42016";
-    case AOM_IMG_FMT_I42216: return "I42216";
-    case AOM_IMG_FMT_I44416: return "I44416";
-    default: return "Other";
-  }
-}
-
 static void show_stream_config(struct stream_state *stream,
                                struct AvxEncoderConfig *global,
                                struct AvxInputContext *input) {
diff --git a/common/tools_common.c b/common/tools_common.c
index 226c35e..114e635 100644
--- a/common/tools_common.c
+++ b/common/tools_common.c
@@ -70,6 +70,21 @@
   exit(EXIT_FAILURE);
 }
 
+const char *image_format_to_string(aom_img_fmt_t fmt) {
+  switch (fmt) {
+    case AOM_IMG_FMT_I420: return "I420";
+    case AOM_IMG_FMT_I422: return "I422";
+    case AOM_IMG_FMT_I444: return "I444";
+    case AOM_IMG_FMT_YV12: return "YV12";
+    case AOM_IMG_FMT_NV12: return "NV12";
+    case AOM_IMG_FMT_YV1216: return "YV1216";
+    case AOM_IMG_FMT_I42016: return "I42016";
+    case AOM_IMG_FMT_I42216: return "I42216";
+    case AOM_IMG_FMT_I44416: return "I44416";
+    default: return "Other";
+  }
+}
+
 int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame) {
   FILE *f = input_ctx->file;
   struct FileTypeDetectionBuffer *detect = &input_ctx->detect;
diff --git a/common/tools_common.h b/common/tools_common.h
index b0b2367..b31371c 100644
--- a/common/tools_common.h
+++ b/common/tools_common.h
@@ -173,6 +173,8 @@
 // If the interface is unknown, returns 0.
 uint32_t get_fourcc_by_aom_decoder(aom_codec_iface_t *iface);
 
+const char *image_format_to_string(aom_img_fmt_t fmt);
+
 int read_yuv_frame(struct AvxInputContext *input_ctx, aom_image_t *yuv_frame);
 
 void aom_img_write(const aom_image_t *img, FILE *file);