validate_img,validate_hbd_input: fix crash w/monochrome Use the monochrome setting from the encoder configuration to determine whether to check the U/V planes. Previously the value from `aom_image_t` was used. Not all applications may set this value. Bug: 513603812 Change-Id: I7fd5d37f2ad863b6392d508f5ad542b68b9c7c88 (cherry picked from commit 13cbb78a1a9ebb5193ef8fae664a507efb4287e5)
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index 77e48de..44482f5 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -1025,9 +1025,10 @@ const int max_val = 1 << bit_depth; // Note there is no high bitdepth version of NV12 defined. If one is // added, `num_planes` should be 2 in that case. - const int num_planes = img->monochrome ? 1 : 3; + const int num_planes = ctx->cfg.monochrome ? 1 : 3; for (int plane = 0; plane < num_planes; ++plane) { const unsigned short *src = (const unsigned short *)img->planes[plane]; + if (!src) return AOM_CODEC_INVALID_PARAM; const unsigned int stride = img->stride[plane] / 2; const unsigned int ph = aom_img_plane_height(img, plane); const unsigned int pw = aom_img_plane_width(img, plane);
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc index ba482bc..d6b3c8d 100644 --- a/test/encode_api_test.cc +++ b/test/encode_api_test.cc
@@ -390,8 +390,10 @@ ++check_input_range) { uint8_t *u = img->planes[AOM_PLANE_U]; uint8_t *v = img->planes[AOM_PLANE_V]; - img->monochrome = cfg.monochrome; - if (img->monochrome) { + // Applications may not set img->monochrome. Leave it set to 0, but + // clear the planes to ensure the encoder properly ignores their + // values. + if (cfg.monochrome) { img->planes[AOM_PLANE_U] = img->planes[AOM_PLANE_V] = nullptr; }