Make avifAlloc(0) deterministic and fix RWData shrink-to-zero handling (#3212)

include/avif/avif.h

diff --git a/include/avif/avif.h b/include/avif/avif.h
index 204827b..e7e14f9 100644
--- a/include/avif/avif.h
+++ b/include/avif/avif.h
@@ -154,7 +154,7 @@
 // ---------------------------------------------------------------------------
 // Memory management
 
-// Returns NULL on memory allocation failure.
+// Returns NULL on memory allocation failure or if size is 0.
 AVIF_API void * avifAlloc(size_t size);
 AVIF_API void avifFree(void * p);
 

src/mem.c

diff --git a/src/mem.c b/src/mem.c
index 5ccf5f3..54d9495 100644
--- a/src/mem.c
+++ b/src/mem.c
@@ -3,12 +3,17 @@
 
 #include "avif/avif.h"
 
-#include <assert.h>
 #include <stdlib.h>
 
 void * avifAlloc(size_t size)
 {
-    assert(size != 0); // Implementation-defined. See https://en.cppreference.com/w/cpp/memory/c/malloc
+    // malloc(0) is implementation-defined (see
+    // https://en.cppreference.com/w/cpp/memory/c/malloc), so collapse the
+    // zero-size case to a deterministic NULL return. Callers must either treat
+    // 0 as an allocation failure or guard against it before calling.
+    if (size == 0) {
+        return NULL;
+    }
     return malloc(size);
 }
 

src/rawdata.c

diff --git a/src/rawdata.c b/src/rawdata.c
index 6f8ff67..7ffdaf6 100644
--- a/src/rawdata.c
+++ b/src/rawdata.c
@@ -8,9 +8,14 @@
 avifResult avifRWDataRealloc(avifRWData * raw, size_t newSize)
 {
     if (raw->size != newSize) {
+        if (newSize == 0) {
+            // avifAlloc(0) returns NULL, so handle the shrink-to-zero case by freeing the buffer.
+            avifRWDataFree(raw);
+            return AVIF_RESULT_OK;
+        }
         uint8_t * newData = (uint8_t *)avifAlloc(newSize);
         AVIF_CHECKERR(newData, AVIF_RESULT_OUT_OF_MEMORY);
-        if (raw->size && newSize) {
+        if (raw->size) {
             memcpy(newData, raw->data, AVIF_MIN(raw->size, newSize));
         }
         avifFree(raw->data);

tests/gtest/avifallocationtest.cc

diff --git a/tests/gtest/avifallocationtest.cc b/tests/gtest/avifallocationtest.cc
index 8854ba3..9668e18 100644
--- a/tests/gtest/avifallocationtest.cc
+++ b/tests/gtest/avifallocationtest.cc
@@ -208,5 +208,27 @@
 }
 #endif
 
+TEST(AvifAllocTest, ZeroReturnsNull) {
+  // avifAlloc(0) must return NULL deterministically (rather than aborting via
+  // assert or relying on implementation-defined malloc(0) behavior). Callers
+  // rely on this contract to treat 0 as an allocation failure.
+  EXPECT_EQ(avifAlloc(0), nullptr);
+}
+
+TEST(AvifAllocTest, RWDataReallocShrinkToZeroSucceeds) {
+  // Shrinking an avifRWData to size 0 must release the buffer and return OK,
+  // not surface a phantom OOM from avifAlloc(0) returning NULL.
+  avifRWData raw = {};
+  ASSERT_EQ(avifRWDataRealloc(&raw, 16), AVIF_RESULT_OK);
+  ASSERT_NE(raw.data, nullptr);
+  ASSERT_EQ(raw.size, 16u);
+
+  EXPECT_EQ(avifRWDataRealloc(&raw, 0), AVIF_RESULT_OK);
+  EXPECT_EQ(raw.data, nullptr);
+  EXPECT_EQ(raw.size, 0u);
+
+  avifRWDataFree(&raw);
+}
+
 }  // namespace
 }  // namespace avif