Check PSNR in transformtest because of rav1e (#2836)

diff --git a/tests/gtest/aviftest_helpers.cc b/tests/gtest/aviftest_helpers.cc
index 6ac8f9b..34357bf 100644
--- a/tests/gtest/aviftest_helpers.cc
+++ b/tests/gtest/aviftest_helpers.cc
@@ -259,11 +259,10 @@
   }
   return true;
 }
-}  // namespace
 
-// Returns true if image1 and image2 are identical.
-bool AreImagesEqual(const avifImage& image1, const avifImage& image2,
-                    bool ignore_alpha) {
+// Returns true if image1 and image2 are identical, pixel values excepted.
+bool AreImageFeaturesEqual(const avifImage& image1, const avifImage& image2,
+                           bool ignore_alpha) {
   if (image1.width != image2.width || image1.height != image2.height ||
       image1.depth != image2.depth || image1.yuvFormat != image2.yuvFormat ||
       image1.yuvRange != image2.yuvRange) {
@@ -278,7 +277,7 @@
     const uint8_t* row2 = avifImagePlane(&image2, c);
     if (!row1 != !row2) {
       // Maybe one image contains an opaque alpha channel while the other has no
-      // alpha channel, but they should still be considered equal.
+      // alpha channel, but the features should still be considered equal.
       if (c == AVIF_CHAN_A && avifImageIsOpaque(&image1) &&
           avifImageIsOpaque(&image2)) {
         continue;
@@ -291,26 +290,6 @@
       // Alpha premultiplication is ignored if alpha is opaque.
       return false;
     }
-    const uint32_t row_bytes1 = avifImagePlaneRowBytes(&image1, c);
-    const uint32_t row_bytes2 = avifImagePlaneRowBytes(&image2, c);
-    const uint32_t plane_width = avifImagePlaneWidth(&image1, c);
-    // 0 for A if no alpha and 0 for UV if 4:0:0.
-    const uint32_t plane_height = avifImagePlaneHeight(&image1, c);
-    for (uint32_t y = 0; y < plane_height; ++y) {
-      if (avifImageUsesU16(&image1)) {
-        if (!std::equal(reinterpret_cast<const uint16_t*>(row1),
-                        reinterpret_cast<const uint16_t*>(row1) + plane_width,
-                        reinterpret_cast<const uint16_t*>(row2))) {
-          return false;
-        }
-      } else {
-        if (!std::equal(row1, row1 + plane_width, row2)) {
-          return false;
-        }
-      }
-      row1 += row_bytes1;
-      row2 += row_bytes2;
-    }
   }
 
   if (!AreByteSequencesEqual(image1.icc, image2.icc)) return false;
@@ -353,10 +332,70 @@
     }
 
     if (!image1.gainMap->image != !image2.gainMap->image) return false;
-    if (image1.gainMap->image != nullptr &&
-        !AreImagesEqual(*image1.gainMap->image, *image2.gainMap->image)) {
-      return false;
+  }
+  return true;
+}
+}  // namespace
+
+// Returns true if image1 and image2 are identical.
+bool AreImagesEqual(const avifImage& image1, const avifImage& image2,
+                    bool ignore_alpha) {
+  if (!AreImageFeaturesEqual(image1, image2, ignore_alpha)) {
+    return false;
+  }
+
+  for (avifChannelIndex c :
+       {AVIF_CHAN_Y, AVIF_CHAN_U, AVIF_CHAN_V, AVIF_CHAN_A}) {
+    if (ignore_alpha && c == AVIF_CHAN_A) continue;
+    const uint8_t* row1 = avifImagePlane(&image1, c);
+    const uint8_t* row2 = avifImagePlane(&image2, c);
+    if (row1 == nullptr || row2 == nullptr) {
+      continue;  // Verified in AreImageFeaturesEqual().
     }
+    const uint32_t row_bytes1 = avifImagePlaneRowBytes(&image1, c);
+    const uint32_t row_bytes2 = avifImagePlaneRowBytes(&image2, c);
+    const uint32_t plane_width = avifImagePlaneWidth(&image1, c);
+    // 0 for A if no alpha and 0 for UV if 4:0:0.
+    const uint32_t plane_height = avifImagePlaneHeight(&image1, c);
+    for (uint32_t y = 0; y < plane_height; ++y) {
+      if (avifImageUsesU16(&image1)) {
+        if (!std::equal(reinterpret_cast<const uint16_t*>(row1),
+                        reinterpret_cast<const uint16_t*>(row1) + plane_width,
+                        reinterpret_cast<const uint16_t*>(row2))) {
+          return false;
+        }
+      } else {
+        if (!std::equal(row1, row1 + plane_width, row2)) {
+          return false;
+        }
+      }
+      row1 += row_bytes1;
+      row2 += row_bytes2;
+    }
+  }
+
+  if (image1.gainMap != nullptr && image1.gainMap->image != nullptr &&
+      image2.gainMap != nullptr && image2.gainMap->image != nullptr &&
+      !AreImagesEqual(*image1.gainMap->image, *image2.gainMap->image)) {
+    return false;
+  }
+  return true;
+}
+
+bool AreImagesSimilar(const avifImage& image1, const avifImage& image2,
+                      double min_psnr, bool ignore_alpha) {
+  if (!AreImageFeaturesEqual(image1, image2, ignore_alpha)) {
+    return false;
+  }
+
+  if (GetPsnr(image1, image2, ignore_alpha) < min_psnr) {
+    return true;
+  }
+
+  if (image1.gainMap != nullptr && image1.gainMap->image != nullptr &&
+      image2.gainMap != nullptr && image2.gainMap->image != nullptr &&
+      GetPsnr(*image1.gainMap->image, *image2.gainMap->image) < min_psnr) {
+    return false;
   }
   return true;
 }
diff --git a/tests/gtest/aviftest_helpers.h b/tests/gtest/aviftest_helpers.h
index 4adfb59..1caf4da 100644
--- a/tests/gtest/aviftest_helpers.h
+++ b/tests/gtest/aviftest_helpers.h
@@ -137,6 +137,10 @@
 // Returns true if both images have the same features and pixel values.
 bool AreImagesEqual(const avifRGBImage& image1, const avifRGBImage& image2);
 
+// Returns true if both images have the same features and close pixel values.
+bool AreImagesSimilar(const avifImage& image1, const avifImage& image2,
+                      double min_psnr = 30, bool ignore_alpha = false);
+
 // Returns the Peak Signal-to-Noise Ratio of image1 compared to image2.
 // A value of 99dB means all samples are exactly the same.
 // A negative value means that the input images cannot be compared.
diff --git a/tests/gtest/aviftransformtest.cc b/tests/gtest/aviftransformtest.cc
index 163378e..e94eef4 100644
--- a/tests/gtest/aviftransformtest.cc
+++ b/tests/gtest/aviftransformtest.cc
@@ -122,7 +122,9 @@
   ASSERT_EQ(avifDecoderReadMemory(decoder.get(), decoded.get(), encoded.data,
                                   encoded.size),
             AVIF_RESULT_OK);
-  EXPECT_TRUE(testutil::AreImagesEqual(*image, *decoded));
+  // Note that rav1e does not support lossless encoding as of v0.8.0.
+  // Otherwise AreImagesEqual() could be used here.
+  EXPECT_TRUE(testutil::AreImagesSimilar(*image, *decoded));
 
   // Check with existing correct AVIF file.
   const std::string ref_path =
@@ -134,7 +136,9 @@
   ASSERT_EQ(avifDecoderReadFile(ref_decoder.get(), ref_decoded.get(),
                                 ref_path.c_str()),
             AVIF_RESULT_OK);
-  EXPECT_TRUE(testutil::AreImagesEqual(*decoded, *ref_decoded));
+  // Note that rav1e does not support lossless encoding as of v0.8.0.
+  // Otherwise AreImagesEqual() could be used here.
+  EXPECT_TRUE(testutil::AreImagesSimilar(*decoded, *ref_decoded));
 
   // The rendering of the images should be compared but that is outside the
   // scope of libavif.