commit b7709140a9818dbe7bc2f3609479043d3e1e968d
author Vincent Rabaud <vrabaud@users.noreply.github.com> Tue May 23 18:47:47 2023 +0200
committer GitHub <noreply@github.com> Tue May 23 09:47:47 2023 -0700
tree 3f3799842efad6105ef014cb70a8273e22f0bba3
parent 6c5dec5a1ca3c2a416b59b08b248be9e8bf418eb

Check for YCgCo-R depth coherence at lowest level

Check for YCgCo-R depths compatibility in avifPrepareReformatState

src/reformat.c

diff --git a/src/reformat.c b/src/reformat.c
index 7a24fce..c6d131f 100644
--- a/src/reformat.c
+++ b/src/reformat.c
@@ -23,12 +23,25 @@
 
 static avifBool avifPrepareReformatState(const avifImage * image, const avifRGBImage * rgb, avifReformatState * state)
 {
+#if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
+    const avifBool useYCgCoRe = (image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RE);
+    const avifBool useYCgCoRo = (image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RO);
+#else
+    const avifBool useYCgCoRe = AVIF_FALSE;
+    const avifBool useYCgCoRo = AVIF_FALSE;
+#endif
     if ((image->depth != 8) && (image->depth != 10) && (image->depth != 12) && (image->depth != 16)) {
         return AVIF_FALSE;
     }
     if ((rgb->depth != 8) && (rgb->depth != 10) && (rgb->depth != 12) && (rgb->depth != 16)) {
         return AVIF_FALSE;
     }
+    if (useYCgCoRe || useYCgCoRo) {
+        const int bitOffset = (useYCgCoRe) ? 2 : 1;
+        if (image->depth - bitOffset != rgb->depth) {
+            return AVIF_FALSE;
+        }
+    }
     if (rgb->isFloat && rgb->depth != 16) {
         return AVIF_FALSE;
     }
@@ -47,13 +60,6 @@
     //
     // YCgCo performs limited-full range adjustment on R,G,B but the current implementation performs range adjustment
     // on Y,U,V. So YCgCo with limited range is unsupported.
-#if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
-    const avifBool useYCgCoRe = (image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RE);
-    const avifBool useYCgCoRo = (image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RO);
-#else
-    const avifBool useYCgCoRe = AVIF_FALSE;
-    const avifBool useYCgCoRo = AVIF_FALSE;
-#endif
     if ((image->matrixCoefficients == 3 /* CICP reserved */) ||
         ((image->matrixCoefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO || useYCgCoRe || useYCgCoRo) &&
          (image->yuvRange == AVIF_RANGE_LIMITED)) ||
@@ -736,16 +742,13 @@
                     R = t + Cr;
 #if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
                 } else if (state->mode == AVIF_REFORMAT_MODE_YCGCO_RE || state->mode == AVIF_REFORMAT_MODE_YCGCO_RO) {
-                    const int bitOffset = (state->mode == AVIF_REFORMAT_MODE_YCGCO_RE) ? 2 : 1;
-                    const int maxValue = (1 << (state->yuvDepth - bitOffset)) - 1;
-                    assert((float)maxValue == rgbMaxChannelF);
                     const int YY = unormY;
                     const int Cg = (int)avifRoundf(Cb * yuvMaxChannel);
                     const int Co = (int)avifRoundf(Cr * yuvMaxChannel);
                     const int t = YY - (Cg >> 1);
-                    G = AVIF_CLAMP(t + Cg, 0, maxValue);
-                    B = AVIF_CLAMP(t - (Co >> 1), 0, maxValue);
-                    R = AVIF_CLAMP(B + Co, 0, maxValue);
+                    G = AVIF_CLAMP(t + Cg, 0, state->rgbMaxChannel);
+                    B = AVIF_CLAMP(t - (Co >> 1), 0, state->rgbMaxChannel);
+                    R = AVIF_CLAMP(B + Co, 0, state->rgbMaxChannel);
                     G /= rgbMaxChannelF;
                     B /= rgbMaxChannelF;
                     R /= rgbMaxChannelF;