Detect encoder and image high bit depth mismatch

If the encoder is initialized with the AOM_CODEC_USE_HIGHBITDEPTH flag,
then the input image must have a high bit depth format. Similarly, if
the encoder is not initialized with the AOM_CODEC_USE_HIGHBITDEPTH flag,
then the input image must not have a high bit depth format.

Change AV1's encoder algorithm to not advertise
AOM_CODEC_CAP_HIGHBITDEPTH if CONFIG_AV1_HIGHBITDEPTH is 0. Change
aom_codec_enc_init_ver() to fail with AOM_CODEC_INCAPABLE if the
AOM_CODEC_USE_HIGHBITDEPTH flag is specified but the encoder algorithm
doesn't have the AOM_CODEC_CAP_HIGHBITDEPTH capability.

Bug: aomedia:3489
Change-Id: Ie08f9ec132b5cb85f1465090f6bc771e5ba1e782
diff --git a/aom/src/aom_encoder.c b/aom/src/aom_encoder.c
index a4acbcc..70e0b75 100644
--- a/aom/src/aom_encoder.c
+++ b/aom/src/aom_encoder.c
@@ -67,7 +67,11 @@
     res = AOM_CODEC_INCAPABLE;
   else if ((flags & AOM_CODEC_USE_PSNR) && !(iface->caps & AOM_CODEC_CAP_PSNR))
     res = AOM_CODEC_INCAPABLE;
-  else if (cfg->g_bit_depth > 8 && (flags & AOM_CODEC_USE_HIGHBITDEPTH) == 0) {
+  else if ((flags & AOM_CODEC_USE_HIGHBITDEPTH) &&
+           !(iface->caps & AOM_CODEC_CAP_HIGHBITDEPTH)) {
+    res = AOM_CODEC_INCAPABLE;
+  } else if (cfg->g_bit_depth > 8 &&
+             (flags & AOM_CODEC_USE_HIGHBITDEPTH) == 0) {
     res = AOM_CODEC_INVALID_PARAM;
     ctx->err_detail =
         "High bit-depth used without the AOM_CODEC_USE_HIGHBITDEPTH flag.";
@@ -171,7 +175,10 @@
     res = AOM_CODEC_ERROR;
   else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER))
     res = AOM_CODEC_INCAPABLE;
-  else {
+  else if (img && ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH) != 0) !=
+                      ((ctx->init_flags & AOM_CODEC_USE_HIGHBITDEPTH) != 0)) {
+    res = AOM_CODEC_INVALID_PARAM;
+  } else {
     /* Execute in a normalized floating point environment, if the platform
      * requires it.
      */
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index a13b09e..f8f71b4 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -4592,11 +4592,11 @@
 aom_codec_iface_t aom_codec_av1_cx_algo = {
   "AOMedia Project AV1 Encoder" VERSION_STRING,
   AOM_CODEC_INTERNAL_ABI_VERSION,
-  AOM_CODEC_CAP_HIGHBITDEPTH | AOM_CODEC_CAP_ENCODER |
-      AOM_CODEC_CAP_PSNR,  // aom_codec_caps_t
-  encoder_init,            // aom_codec_init_fn_t
-  encoder_destroy,         // aom_codec_destroy_fn_t
-  encoder_ctrl_maps,       // aom_codec_ctrl_fn_map_t
+  (CONFIG_AV1_HIGHBITDEPTH ? AOM_CODEC_CAP_HIGHBITDEPTH : 0) |
+      AOM_CODEC_CAP_ENCODER | AOM_CODEC_CAP_PSNR,  // aom_codec_caps_t
+  encoder_init,                                    // aom_codec_init_fn_t
+  encoder_destroy,                                 // aom_codec_destroy_fn_t
+  encoder_ctrl_maps,                               // aom_codec_ctrl_fn_map_t
   {
       // NOLINT
       NULL,  // aom_codec_peek_si_fn_t
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 9e6e9c5..bc5ce24 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -10,6 +10,7 @@
  */
 
 #include <cstdlib>
+#include <cstring>
 
 #include "third_party/googletest/src/googletest/include/gtest/gtest.h"
 
@@ -27,6 +28,12 @@
 const int kUsage = AOM_USAGE_GOOD_QUALITY;
 #endif
 
+static void *Memset16(void *dest, int val, size_t length) {
+  uint16_t *dest16 = (uint16_t *)dest;
+  for (size_t i = 0; i < length; ++i) *dest16++ = val;
+  return dest;
+}
+
 TEST(EncodeAPI, InvalidParams) {
   uint8_t buf[1] = { 0 };
   aom_image_t img;
@@ -130,6 +137,146 @@
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
 }
 
+TEST(EncodeAPI, LowBDEncoderLowBDImage) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, kUsage), AOM_CODEC_OK);
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+
+  aom_image_t *image =
+      aom_img_alloc(NULL, AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h, 0);
+  ASSERT_NE(image, nullptr);
+
+  // Set the image to two colors so that av1_set_screen_content_options() will
+  // call av1_get_perpixel_variance().
+  int luma_value = 0;
+  for (unsigned int i = 0; i < image->d_h; ++i) {
+    memset(image->planes[0] + i * image->stride[0], luma_value, image->d_w);
+    luma_value = 255 - luma_value;
+  }
+  unsigned int uv_h = (image->d_h + 1) / 2;
+  unsigned int uv_w = (image->d_w + 1) / 2;
+  for (unsigned int i = 0; i < uv_h; ++i) {
+    memset(image->planes[1] + i * image->stride[1], 128, uv_w);
+    memset(image->planes[2] + i * image->stride[2], 128, uv_w);
+  }
+
+  ASSERT_EQ(aom_codec_encode(&enc, image, 0, 1, 0), AOM_CODEC_OK);
+
+  aom_img_free(image);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
+TEST(EncodeAPI, HighBDEncoderHighBDImage) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, kUsage), AOM_CODEC_OK);
+
+  aom_codec_ctx_t enc;
+  aom_codec_err_t init_status =
+      aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_HIGHBITDEPTH);
+#if !CONFIG_AV1_HIGHBITDEPTH
+  ASSERT_EQ(init_status, AOM_CODEC_INCAPABLE);
+#else
+  ASSERT_EQ(init_status, AOM_CODEC_OK);
+
+  aom_image_t *image =
+      aom_img_alloc(NULL, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
+  ASSERT_NE(image, nullptr);
+
+  // Set the image to two colors so that av1_set_screen_content_options() will
+  // call av1_get_perpixel_variance().
+  int luma_value = 0;
+  for (unsigned int i = 0; i < image->d_h; ++i) {
+    Memset16(image->planes[0] + i * image->stride[0], luma_value, image->d_w);
+    luma_value = 255 - luma_value;
+  }
+  unsigned int uv_h = (image->d_h + 1) / 2;
+  unsigned int uv_w = (image->d_w + 1) / 2;
+  for (unsigned int i = 0; i < uv_h; ++i) {
+    Memset16(image->planes[1] + i * image->stride[1], 128, uv_w);
+    Memset16(image->planes[2] + i * image->stride[2], 128, uv_w);
+  }
+
+  ASSERT_EQ(aom_codec_encode(&enc, image, 0, 1, 0), AOM_CODEC_OK);
+
+  aom_img_free(image);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+#endif
+}
+
+TEST(EncodeAPI, HighBDEncoderLowBDImage) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, kUsage), AOM_CODEC_OK);
+
+  aom_codec_ctx_t enc;
+  aom_codec_err_t init_status =
+      aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_HIGHBITDEPTH);
+#if !CONFIG_AV1_HIGHBITDEPTH
+  ASSERT_EQ(init_status, AOM_CODEC_INCAPABLE);
+#else
+  ASSERT_EQ(init_status, AOM_CODEC_OK);
+
+  aom_image_t *image =
+      aom_img_alloc(NULL, AOM_IMG_FMT_I420, cfg.g_w, cfg.g_h, 0);
+  ASSERT_NE(image, nullptr);
+
+  // Set the image to two colors so that av1_set_screen_content_options() will
+  // call av1_get_perpixel_variance().
+  int luma_value = 0;
+  for (unsigned int i = 0; i < image->d_h; ++i) {
+    memset(image->planes[0] + i * image->stride[0], luma_value, image->d_w);
+    luma_value = 255 - luma_value;
+  }
+  unsigned int uv_h = (image->d_h + 1) / 2;
+  unsigned int uv_w = (image->d_w + 1) / 2;
+  for (unsigned int i = 0; i < uv_h; ++i) {
+    memset(image->planes[1] + i * image->stride[1], 128, uv_w);
+    memset(image->planes[2] + i * image->stride[2], 128, uv_w);
+  }
+
+  ASSERT_EQ(aom_codec_encode(&enc, image, 0, 1, 0), AOM_CODEC_INVALID_PARAM);
+
+  aom_img_free(image);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+#endif
+}
+
+TEST(EncodeAPI, LowBDEncoderHighBDImage) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, kUsage), AOM_CODEC_OK);
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+
+  aom_image_t *image =
+      aom_img_alloc(NULL, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 0);
+  ASSERT_NE(image, nullptr);
+
+  // Set the image to two colors so that av1_set_screen_content_options() will
+  // call av1_get_perpixel_variance().
+  int luma_value = 0;
+  for (unsigned int i = 0; i < image->d_h; ++i) {
+    Memset16(image->planes[0] + i * image->stride[0], luma_value, image->d_w);
+    luma_value = 255 - luma_value;
+  }
+  unsigned int uv_h = (image->d_h + 1) / 2;
+  unsigned int uv_w = (image->d_w + 1) / 2;
+  for (unsigned int i = 0; i < uv_h; ++i) {
+    Memset16(image->planes[1] + i * image->stride[1], 128, uv_w);
+    Memset16(image->planes[2] + i * image->stride[2], 128, uv_w);
+  }
+
+  ASSERT_EQ(aom_codec_encode(&enc, image, 0, 1, 0), AOM_CODEC_INVALID_PARAM);
+
+  aom_img_free(image);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
 #if !CONFIG_REALTIME_ONLY
 TEST(EncodeAPI, AllIntraMode) {
   aom_codec_iface_t *iface = aom_codec_av1_cx();