commit 8f8ed4f476ec77fbea3142c934ca416a78e05bf8
author Joe Drago <jdrago@netflix.com> Fri Sep 25 11:35:44 2020 -0700
committer Joe Drago <jdrago@netflix.com> Fri Sep 25 11:35:44 2020 -0700
tree f070da7ca60c2d76fbd8dda9db9ce4466775cf78
parent 76556c4c24cb935039e1ba87006d80fb513a8d7b

Fix an infinite loop in codec_dav1d on a carefully-malformed AV1 payload

src/codec_dav1d.c

diff --git a/src/codec_dav1d.c b/src/codec_dav1d.c
index fcd4af8..16afb1d 100644
--- a/src/codec_dav1d.c
+++ b/src/codec_dav1d.c
@@ -29,7 +29,6 @@
     Dav1dPicture dav1dPicture;
     avifBool hasPicture;
     avifRange colorRange;
-    Dav1dData dav1dData;
 };
 
 static void avifDav1dFreeCallback(const uint8_t * buf, void * cookie)
@@ -41,9 +40,6 @@
 
 static void dav1dCodecDestroyInternal(avifCodec * codec)
 {
-    if (codec->internal->dav1dData.sz) {
-        dav1d_data_unref(&codec->internal->dav1dData);
-    }
     if (codec->internal->hasPicture) {
         dav1d_picture_unref(&codec->internal->dav1dPicture);
     }
@@ -53,27 +49,6 @@
     avifFree(codec->internal);
 }
 
-// returns AVIF_FALSE if there's nothing left to feed, or feeding fatally fails (say that five times fast)
-static avifBool dav1dFeedData(avifCodec * codec, avifDecodeSample * sample)
-{
-    if (!codec->internal->dav1dData.sz) {
-        if (sample) {
-            if (dav1d_data_wrap(&codec->internal->dav1dData, sample->data.data, sample->data.size, avifDav1dFreeCallback, NULL) != 0) {
-                return AVIF_FALSE;
-            }
-        } else {
-            // No more data
-            return AVIF_FALSE;
-        }
-    }
-
-    int res = dav1d_send_data(codec->internal->dav1dContext, &codec->internal->dav1dData);
-    if ((res < 0) && (res != DAV1D_ERR(EAGAIN))) {
-        return AVIF_FALSE;
-    }
-    return AVIF_TRUE;
-}
-
 static avifBool dav1dCodecOpen(avifCodec * codec)
 {
     if (codec->internal->dav1dContext == NULL) {
@@ -90,14 +65,32 @@
     Dav1dPicture nextFrame;
     memset(&nextFrame, 0, sizeof(Dav1dPicture));
 
+    Dav1dData dav1dData;
+    if (dav1d_data_wrap(&dav1dData, sample->data.data, sample->data.size, avifDav1dFreeCallback, NULL) != 0) {
+        return AVIF_FALSE;
+    }
+
     for (;;) {
-        avifBool sentData = dav1dFeedData(codec, sample);
+        if (dav1dData.data) {
+            int res = dav1d_send_data(codec->internal->dav1dContext, &dav1dData);
+            if ((res < 0) && (res != DAV1D_ERR(EAGAIN))) {
+                dav1d_data_unref(&dav1dData);
+                return AVIF_FALSE;
+            }
+        }
+
         int res = dav1d_get_picture(codec->internal->dav1dContext, &nextFrame);
-        if ((res == DAV1D_ERR(EAGAIN)) && sentData) {
-            // send more data
-            continue;
+        if (res == DAV1D_ERR(EAGAIN)) {
+            if (dav1dData.data) {
+                // send more data
+                continue;
+            }
+            return AVIF_FALSE;
         } else if (res < 0) {
             // No more frames
+            if (dav1dData.data) {
+                dav1d_data_unref(&dav1dData);
+            }
             return AVIF_FALSE;
         } else {
             // Got a picture!
@@ -105,6 +98,9 @@
             break;
         }
     }
+    if (dav1dData.data) {
+        dav1d_data_unref(&dav1dData);
+    }
 
     if (gotPicture) {
         dav1d_picture_unref(&codec->internal->dav1dPicture);