Guard JPEG XMP parsing sizes (#3249)

apps/shared/avifjpeg.c

diff --git a/apps/shared/avifjpeg.c b/apps/shared/avifjpeg.c
index 4a557cb..c028575 100644
--- a/apps/shared/avifjpeg.c
+++ b/apps/shared/avifjpeg.c
@@ -7,6 +7,7 @@
 
 #include <assert.h>
 #include <ctype.h>
+#include <limits.h>
 #include <math.h>
 #include <setjmp.h>
 #include <stdint.h>
@@ -579,6 +580,14 @@
 #define XML_NAME_SPACE_RDF "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
 #define XML_NAME_SPACE_XMP_NOTE "http://ns.adobe.com/xmp/note/"
 
+static xmlDoc * avifJPEGReadXMLMemory(const uint8_t * data, size_t size, const char * url)
+{
+    if (size > INT_MAX) {
+        return NULL;
+    }
+    return xmlReadMemory((const char *)data, (int)size, url, NULL, /*options=*/0);
+}
+
 // Finds an 'rdf:Description' node containing a gain map version attribute (hdrgm:Version="1.0").
 // Returns NULL if not found.
 static const xmlNode * avifJPEGFindIsoGainMapXMPNode(const xmlNode * rootNode)
@@ -664,7 +673,7 @@
 // If not null, isAppleGainMap is set to AVIF_TRUE for an Apple style gain map, and AVIF_FALSE for an ISO gain map.
 static avifBool avifJPEGHasGainMapXMPNode(const uint8_t * xmpData, size_t xmpSize, avifBool * isAppleGainMap)
 {
-    xmlDoc * document = xmlReadMemory((const char *)xmpData, (int)xmpSize, NULL, NULL, /*options=*/0);
+    xmlDoc * document = avifJPEGReadXMLMemory(xmpData, xmpSize, NULL);
     if (document == NULL) {
         return AVIF_FALSE; // Probably and out of memory error.
     }
@@ -885,7 +894,7 @@
 // Returns AVIF_TRUE if the gain map metadata was successfully read.
 avifBool avifJPEGParseGainMapXMP(const uint8_t * xmpData, size_t xmpSize, avifGainMap * gainMap, avifBool * isAppleGainMap)
 {
-    xmlDoc * document = xmlReadMemory((const char *)xmpData, (int)xmpSize, NULL, NULL, /*options=*/0);
+    xmlDoc * document = avifJPEGReadXMLMemory(xmpData, xmpSize, NULL);
     if (document == NULL) {
         return AVIF_FALSE; // Probably an out of memory error.
     }
@@ -1122,7 +1131,12 @@
     avifBool isValid = AVIF_TRUE;
     xmlDoc * extendedXMPDoc = NULL;
     xmlChar * xmlBuff = NULL;
-    xmlDoc * xmpDoc = xmlReadMemory((const char *)standardXMPData, (int)standardXMPSize, "standard.xml", NULL, /*options=*/0);
+    xmlDoc * xmpDoc = avifJPEGReadXMLMemory(standardXMPData, standardXMPSize, "standard.xml");
+    if (xmpDoc == NULL) {
+        fprintf(stderr, "XMP extraction failed: invalid standard XMP segment\n");
+        isValid = AVIF_FALSE;
+        goto cleanup_xml;
+    }
     xmlNode * xmpRdf = (xmlNode *)avifJPEGFindXMLNodeByName(xmlDocGetRootElement(xmpDoc),
                                                             XML_NAME_SPACE_RDF,
                                                             "RDF",
@@ -1174,11 +1188,12 @@
     }
 
     // Read the extended XMP.
-    extendedXMPDoc = xmlReadMemory((const char *)extendedXMP.data,
-                                   (int)extendedXMP.size,
-                                   "extended.xml",
-                                   NULL,
-                                   /*options=*/0);
+    extendedXMPDoc = avifJPEGReadXMLMemory(extendedXMP.data, extendedXMP.size, "extended.xml");
+    if (extendedXMPDoc == NULL) {
+        fprintf(stderr, "XMP extraction failed: invalid extended XMP segment\n");
+        isValid = AVIF_FALSE;
+        goto cleanup_xml;
+    }
     const xmlNode * extendedXMPRdf = avifJPEGFindXMLNodeByName(xmlDocGetRootElement(extendedXMPDoc),
                                                                XML_NAME_SPACE_RDF,
                                                                "RDF",

tests/gtest/avifjpeggainmaptest.cc

diff --git a/tests/gtest/avifjpeggainmaptest.cc b/tests/gtest/avifjpeggainmaptest.cc
index f3973b2..bb44231 100644
--- a/tests/gtest/avifjpeggainmaptest.cc
+++ b/tests/gtest/avifjpeggainmaptest.cc
@@ -3,6 +3,8 @@
 
 #include <math.h>
 
+#include <limits>
+
 #include "avif/avif.h"
 #include "avifjpeg.h"
 #include "aviftest_helpers.h"
@@ -333,6 +335,15 @@
                                        gain_map.get(), &is_avif_gain_map));
 }
 
+TEST(JpegTest, TooLargeXMP) {
+  const uint8_t xmp = 0;
+  GainMapPtr gain_map(avifGainMapCreate());
+  avifBool is_avif_gain_map;
+  EXPECT_FALSE(avifJPEGParseGainMapXMP(
+      &xmp, static_cast<size_t>(std::numeric_limits<int>::max()) + 1,
+      gain_map.get(), &is_avif_gain_map));
+}
+
 //------------------------------------------------------------------------------
 
 }  // namespace