Check highdepth flag when initializing encoder.

BUG=aomedia:2987

Change-Id: I0a07df3af5a01daab6ae223641921ca189deb5d1

aom/src/aom_encoder.c

diff --git a/aom/src/aom_encoder.c b/aom/src/aom_encoder.c
index bb51c93..5dfda96 100644
--- a/aom/src/aom_encoder.c
+++ b/aom/src/aom_encoder.c
@@ -50,7 +50,11 @@
     res = AOM_CODEC_INCAPABLE;
   else if ((flags & AOM_CODEC_USE_PSNR) && !(iface->caps & AOM_CODEC_CAP_PSNR))
     res = AOM_CODEC_INCAPABLE;
-  else {
+  else if (cfg->g_bit_depth > 8 && (flags & AOM_CODEC_USE_HIGHBITDEPTH) == 0) {
+    res = AOM_CODEC_INVALID_PARAM;
+    ctx->err_detail =
+        "High bit-depth used without the AOM_CODEC_USE_HIGHBITDEPTH flag.";
+  } else {
     ctx->iface = iface;
     ctx->name = iface->name;
     ctx->priv = NULL;