Reject avifRGBImage dimensions mismatch (#3394)

Make the implementation match the documentation.

Add tests.

---------

Co-authored-by: krishna28238-arch <krishna28238-arch@users.noreply.github.com>
Co-authored-by: Yannis Guyon <yguyon@google.com>

src/reformat.c

diff --git a/src/reformat.c b/src/reformat.c
index d1a511b..8ac4945 100644
--- a/src/reformat.c
+++ b/src/reformat.c
@@ -160,6 +160,10 @@
 
 static avifBool avifPrepareReformatState(const avifImage * image, const avifRGBImage * rgb, avifReformatState * state)
 {
+    if (rgb->width != image->width || rgb->height != image->height) {
+        return AVIF_FALSE;
+    }
+
     const avifBool useYCgCoRe = (image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RE);
     const avifBool useYCgCoRo = (image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RO);
     if (useYCgCoRe || useYCgCoRo) {

tests/gtest/avifrgbtoyuvtest.cc

diff --git a/tests/gtest/avifrgbtoyuvtest.cc b/tests/gtest/avifrgbtoyuvtest.cc
index 54dc993..99301c9 100644
--- a/tests/gtest/avifrgbtoyuvtest.cc
+++ b/tests/gtest/avifrgbtoyuvtest.cc
@@ -520,6 +520,35 @@
   avifRGBImageFreePixels(&rgb);
 }
 
+TEST(YUVToRGBTest, DimensionMismatchIsRejected) {
+  ImagePtr image =
+      testutil::CreateImage(/*width=*/4, /*height=*/4, /*depth=*/8,
+                            AVIF_PIXEL_FORMAT_YUV444, AVIF_PLANES_ALL);
+  ASSERT_NE(image, nullptr);
+  testutil::FillImageGradient(image.get());
+
+  testutil::AvifRgbImage rgb(image.get(), /*rgbDepth=*/8, AVIF_RGB_FORMAT_RGBA);
+  rgb.width = image->width - 1;
+  ASSERT_EQ(avifRGBImageAllocatePixels(&rgb), AVIF_RESULT_OK);
+  EXPECT_EQ(avifImageYUVToRGB(image.get(), &rgb), AVIF_RESULT_REFORMAT_FAILED);
+}
+
+TEST(RGBToYUVTest, DimensionMismatchIsRejected) {
+  ImagePtr image =
+      testutil::CreateImage(/*width=*/4, /*height=*/4, /*depth=*/8,
+                            AVIF_PIXEL_FORMAT_YUV444, AVIF_PLANES_ALL);
+  ASSERT_NE(image, nullptr);
+
+  testutil::AvifRgbImage rgb(image.get(), /*rgbDepth=*/8, AVIF_RGB_FORMAT_RGBA);
+  rgb.height = image->height + 1;
+  ASSERT_EQ(avifRGBImageAllocatePixels(&rgb), AVIF_RESULT_OK);
+  for (uint32_t channel_offset : {0, 1, 2, 3}) {
+    testutil::FillImageChannel(&rgb, channel_offset, 123);
+  }
+
+  EXPECT_EQ(avifImageRGBToYUV(image.get(), &rgb), AVIF_RESULT_REFORMAT_FAILED);
+}
+
 //------------------------------------------------------------------------------
 // Selected configurations