Clean YCgCo-R a bit. (#2418)

diff --git a/src/reformat.c b/src/reformat.c
index d59435d..3a34d5f 100644
--- a/src/reformat.c
+++ b/src/reformat.c
@@ -361,7 +361,7 @@
                             yuvBlock[bI][bJ].v = 0.5f * (rgbPixel[0] - rgbPixel[2]);
 #if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
                         } else if (state.yuv.mode == AVIF_REFORMAT_MODE_YCGCO_RE || state.yuv.mode == AVIF_REFORMAT_MODE_YCGCO_RO) {
-                            // Formulas from JVET-U0093.
+                            // Formulas 58,59,60,61 from https://www.itu.int/rec/T-REC-H.273-202407-P
                             const int R = (int)avifRoundf(AVIF_CLAMP(rgbPixel[0] * rgbMaxChannelF, 0.0f, rgbMaxChannelF));
                             const int G = (int)avifRoundf(AVIF_CLAMP(rgbPixel[1] * rgbMaxChannelF, 0.0f, rgbMaxChannelF));
                             const int B = (int)avifRoundf(AVIF_CLAMP(rgbPixel[2] * rgbMaxChannelF, 0.0f, rgbMaxChannelF));
@@ -771,6 +771,7 @@
                     R = t + Cr;
 #if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
                 } else if (state->yuv.mode == AVIF_REFORMAT_MODE_YCGCO_RE || state->yuv.mode == AVIF_REFORMAT_MODE_YCGCO_RO) {
+                    // YCgCoRe/YCgCoRo: Formulas 62,63,64,65 from https://www.itu.int/rec/T-REC-H.273-202407-P
                     const int YY = unormY;
                     const int Cg = (int)avifRoundf(Cb * yuvMaxChannel);
                     const int Co = (int)avifRoundf(Cr * yuvMaxChannel);
diff --git a/tests/gtest/avifrgbtoyuvtest.cc b/tests/gtest/avifrgbtoyuvtest.cc
index d03e5ef..5db8e90 100644
--- a/tests/gtest/avifrgbtoyuvtest.cc
+++ b/tests/gtest/avifrgbtoyuvtest.cc
@@ -358,26 +358,45 @@
         for (avifRange yuv_range : {AVIF_RANGE_LIMITED, AVIF_RANGE_FULL}) {
           for (decltype(AVIF_MATRIX_COEFFICIENTS_IDENTITY) matrix_coefficients :
                {
-                   AVIF_MATRIX_COEFFICIENTS_BT709,
-                   AVIF_MATRIX_COEFFICIENTS_UNSPECIFIED,
-                   AVIF_MATRIX_COEFFICIENTS_FCC,
-                   AVIF_MATRIX_COEFFICIENTS_BT470BG,
-                   AVIF_MATRIX_COEFFICIENTS_BT601,
-                   AVIF_MATRIX_COEFFICIENTS_SMPTE240,
-                   AVIF_MATRIX_COEFFICIENTS_YCGCO,
-                   AVIF_MATRIX_COEFFICIENTS_BT2020_NCL,
-                   AVIF_MATRIX_COEFFICIENTS_CHROMA_DERIVED_NCL
-                   // These are unsupported. See avifPrepareReformatState().
-                   // AVIF_MATRIX_COEFFICIENTS_BT2020_CL
-                   // AVIF_MATRIX_COEFFICIENTS_SMPTE2085
-                   // AVIF_MATRIX_COEFFICIENTS_CHROMA_DERIVED_CL
-                   // AVIF_MATRIX_COEFFICIENTS_ICTCP
+                 AVIF_MATRIX_COEFFICIENTS_BT709,
+                     AVIF_MATRIX_COEFFICIENTS_UNSPECIFIED,
+                     AVIF_MATRIX_COEFFICIENTS_FCC,
+                     AVIF_MATRIX_COEFFICIENTS_BT470BG,
+                     AVIF_MATRIX_COEFFICIENTS_BT601,
+                     AVIF_MATRIX_COEFFICIENTS_SMPTE240,
+                     AVIF_MATRIX_COEFFICIENTS_YCGCO,
+                     AVIF_MATRIX_COEFFICIENTS_BT2020_NCL,
+                     AVIF_MATRIX_COEFFICIENTS_CHROMA_DERIVED_NCL,
+#if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
+                     AVIF_MATRIX_COEFFICIENTS_YCGCO_RE,
+                     AVIF_MATRIX_COEFFICIENTS_YCGCO_RO,
+#endif
+                 // These are unsupported. See avifPrepareReformatState().
+                 // AVIF_MATRIX_COEFFICIENTS_BT2020_CL
+                 // AVIF_MATRIX_COEFFICIENTS_SMPTE2085
+                 // AVIF_MATRIX_COEFFICIENTS_CHROMA_DERIVED_CL
+                 // AVIF_MATRIX_COEFFICIENTS_ICTCP
                }) {
             if (matrix_coefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO &&
                 yuv_range == AVIF_RANGE_LIMITED) {
               // See avifPrepareReformatState().
               continue;
             }
+#if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
+            if ((matrix_coefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RE &&
+                 yuv_depth - 2 != rgb_depth) ||
+                (matrix_coefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RO &&
+                 yuv_depth - 1 != rgb_depth)) {
+              // See avifPrepareReformatState().
+              continue;
+            }
+            if ((matrix_coefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RE ||
+                 matrix_coefficients == AVIF_MATRIX_COEFFICIENTS_YCGCO_RO) &&
+                yuv_range != AVIF_RANGE_FULL) {
+              // YCgCo-R is for lossless.
+              continue;
+            }
+#endif
             for (avifChromaDownsampling chroma_downsampling :
                  {AVIF_CHROMA_DOWNSAMPLING_FASTEST,
                   AVIF_CHROMA_DOWNSAMPLING_BEST_QUALITY}) {
@@ -458,6 +477,10 @@
     AVIF_MATRIX_COEFFICIENTS_BT709;
 constexpr avifMatrixCoefficients kMatrixCoefficientsIdentity =
     AVIF_MATRIX_COEFFICIENTS_IDENTITY;
+#if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
+constexpr avifMatrixCoefficients kMatrixCoefficientsYCgCoRe =
+    AVIF_MATRIX_COEFFICIENTS_YCGCO_RE;
+#endif
 
 // This is the default avifenc setup when encoding from 8b PNG files to AVIF.
 INSTANTIATE_TEST_SUITE_P(
@@ -592,6 +615,22 @@
                                  /*max_abs_average_diff=*/Values(0.),
                                  /*min_psnr=*/Values(99.)));
 
+#if defined(AVIF_ENABLE_EXPERIMENTAL_YCGCO_R)
+// Tests YCGCO_RE is lossless.
+INSTANTIATE_TEST_SUITE_P(YCgCo_Re8b, RGBToYUVTest,
+                         Combine(/*rgb_depth=*/Values(8),
+                                 /*yuv_depth=*/Values(10),
+                                 Values(AVIF_RGB_FORMAT_RGBA),
+                                 Values(AVIF_PIXEL_FORMAT_YUV444),
+                                 Values(AVIF_RANGE_FULL),
+                                 Values(kMatrixCoefficientsYCgCoRe),
+                                 Values(AVIF_CHROMA_DOWNSAMPLING_AUTOMATIC),
+                                 /*add_noise=*/Values(true),
+                                 /*rgb_step=*/Values(101),
+                                 /*max_abs_average_diff=*/Values(0.),
+                                 /*min_psnr=*/Values(99.)));
+#endif
+
 // Coverage for reformat_libsharpyuv.c.
 INSTANTIATE_TEST_SUITE_P(
     SharpYuv8Bit, RGBToYUVTest,