Keyframe info and Nth image support Added: - stss box parsing for keyframe information - avifBool avifDecoderIsKeyframe(avifDecoder * decoder, uint32_t frameIndex); - uint32_t avifDecoderNearestKeyframe(avifDecoder * decoder, uint32_t frameIndex); - avifResult avifDecoderNthImage(avifDecoder * decoder, uint32_t frameIndex); - aviffuzz prints keyframe information as it repeatedly decodes Changed: - internally renamed codec function "decode" to "open", as that's all it does - dav1d codec's open function no longer does an initial unnecessary feed - avifCodecDecodeInput now stores an array of avifSample which know if they're keyframes - moved codec flushing code into avifDecoderFlush() so it is available to avifDecoderNthImage - ptsInTimescales is now calculated independently of frame decode order
diff --git a/CHANGELOG.md b/CHANGELOG.md index 9979fdc..e3308bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -5,6 +5,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- stss box parsing for keyframe information +- avifBool avifDecoderIsKeyframe(avifDecoder * decoder, uint32_t frameIndex); +- uint32_t avifDecoderNearestKeyframe(avifDecoder * decoder, uint32_t frameIndex); +- avifResult avifDecoderNthImage(avifDecoder * decoder, uint32_t frameIndex); +- aviffuzz prints keyframe information as it repeatedly decodes + +### Changed +- internally renamed codec function "decode" to "open", as that's all it does +- dav1d codec's open function no longer does an initial unnecessary feed +- avifCodecDecodeInput now stores an array of avifSample which know if they're keyframes +- moved codec flushing code into avifDecoderFlush() so it is available to avifDecoderNthImage +- ptsInTimescales is now calculated independently of frame decode order ## [0.3.9] - 2019-09-25 ### Changed
diff --git a/include/avif/avif.h b/include/avif/avif.h index 1e0edd1..6e4abc5 100644 --- a/include/avif/avif.h +++ b/include/avif/avif.h
@@ -399,8 +399,15 @@ avifResult avifDecoderSetSource(avifDecoder * decoder, avifDecoderSource source); avifResult avifDecoderParse(avifDecoder * decoder, avifROData * input); avifResult avifDecoderNextImage(avifDecoder * decoder); +avifResult avifDecoderNthImage(avifDecoder * decoder, uint32_t frameIndex); avifResult avifDecoderReset(avifDecoder * decoder); +// Keyframe information +// frameIndex - 0-based, matching avifDecoder->imageIndex, bound by avifDecoder->imageCount +// "nearest" keyframe means the keyframe prior to this frame index (returns frameIndex if it is a keyframe) +avifBool avifDecoderIsKeyframe(avifDecoder * decoder, uint32_t frameIndex); +uint32_t avifDecoderNearestKeyframe(avifDecoder * decoder, uint32_t frameIndex); + // avifEncoder notes: // * if avifEncoderWrite() returns AVIF_RESULT_OK, output must be freed with avifRWDataFree() // * if (maxThreads < 2), multithreading is disabled
diff --git a/include/avif/internal.h b/include/avif/internal.h index 0de7181..5185953 100644 --- a/include/avif/internal.h +++ b/include/avif/internal.h
@@ -72,9 +72,16 @@ // --------------------------------------------------------------------------- // avifCodecDecodeInput +typedef struct avifSample +{ + avifROData data; + avifBool sync; // is sync sample (keyframe) +} avifSample; +AVIF_ARRAY_DECLARE(avifSampleArray, avifSample, sample); + typedef struct avifCodecDecodeInput { - avifRODataArray samples; + avifSampleArray samples; avifBool alpha; // if true, this is decoding an alpha plane } avifCodecDecodeInput; @@ -119,7 +126,7 @@ struct avifCodec; struct avifCodecInternal; -typedef avifBool (*avifCodecDecodeFunc)(struct avifCodec * codec); +typedef avifBool (*avifCodecOpenFunc)(struct avifCodec * codec); // avifCodecAlphaLimitedRangeFunc: returns AVIF_TRUE if an alpha plane exists and was encoded with limited range typedef avifBool (*avifCodecAlphaLimitedRangeFunc)(struct avifCodec * codec); typedef avifBool (*avifCodecGetNextImageFunc)(struct avifCodec * codec, avifImage * image); @@ -133,7 +140,7 @@ avifCodecDecodeInput * decodeInput; struct avifCodecInternal * internal; // up to each codec to use how it wants - avifCodecDecodeFunc decode; + avifCodecOpenFunc open; avifCodecAlphaLimitedRangeFunc alphaLimitedRange; avifCodecGetNextImageFunc getNextImage; avifCodecEncodeImageFunc encodeImage;
diff --git a/src/codec_aom.c b/src/codec_aom.c index fd4c9ea..eb6a8fd 100644 --- a/src/codec_aom.c +++ b/src/codec_aom.c
@@ -48,7 +48,7 @@ avifFree(codec->internal); } -static avifBool aomCodecDecode(struct avifCodec * codec) +static avifBool aomCodecOpen(struct avifCodec * codec) { aom_codec_iface_t * decoder_interface = aom_codec_av1_dx(); if (aom_codec_dec_init(&codec->internal->decoder, decoder_interface, NULL, 0)) { @@ -83,10 +83,10 @@ break; } else if (codec->internal->inputSampleIndex < codec->decodeInput->samples.count) { // Feed another sample - avifROData * sample = &codec->decodeInput->samples.raw[codec->internal->inputSampleIndex]; + avifSample * sample = &codec->decodeInput->samples.sample[codec->internal->inputSampleIndex]; ++codec->internal->inputSampleIndex; codec->internal->iter = NULL; - if (aom_codec_decode(&codec->internal->decoder, sample->data, sample->size, NULL)) { + if (aom_codec_decode(&codec->internal->decoder, sample->data.data, sample->data.size, NULL)) { return AVIF_FALSE; } } else { @@ -427,7 +427,7 @@ { avifCodec * codec = (avifCodec *)avifAlloc(sizeof(avifCodec)); memset(codec, 0, sizeof(struct avifCodec)); - codec->decode = aomCodecDecode; + codec->open = aomCodecOpen; codec->alphaLimitedRange = aomCodecAlphaLimitedRange; codec->getNextImage = aomCodecGetNextImage; codec->encodeImage = aomCodecEncodeImage;
diff --git a/src/codec_dav1d.c b/src/codec_dav1d.c index b48c10f..5d164d7 100644 --- a/src/codec_dav1d.c +++ b/src/codec_dav1d.c
@@ -38,12 +38,12 @@ dav1d_data_unref(&codec->internal->dav1dData); if (codec->internal->inputSampleIndex < codec->decodeInput->samples.count) { - avifROData * sample = &codec->decodeInput->samples.raw[codec->internal->inputSampleIndex]; + avifSample * sample = &codec->decodeInput->samples.sample[codec->internal->inputSampleIndex]; ++codec->internal->inputSampleIndex; // OPTIMIZE: Carefully switch this to use dav1d_data_wrap or dav1d_data_wrap_user_data - uint8_t * dav1dDataPtr = dav1d_data_create(&codec->internal->dav1dData, sample->size); - memcpy(dav1dDataPtr, sample->data, sample->size); + uint8_t * dav1dDataPtr = dav1d_data_create(&codec->internal->dav1dData, sample->data.size); + memcpy(dav1dDataPtr, sample->data.data, sample->data.size); } else { // No more data return AVIF_FALSE; @@ -57,7 +57,7 @@ return AVIF_TRUE; } -static avifBool dav1dCodecDecode(avifCodec * codec) +static avifBool dav1dCodecOpen(avifCodec * codec) { if (codec->internal->dav1dContext == NULL) { if (dav1d_open(&codec->internal->dav1dContext, &codec->internal->dav1dSettings) != 0) { @@ -66,7 +66,7 @@ } codec->internal->inputSampleIndex = 0; - return dav1dFeedData(codec); + return AVIF_TRUE; } static avifBool dav1dCodecAlphaLimitedRange(avifCodec * codec) @@ -184,7 +184,7 @@ { avifCodec * codec = (avifCodec *)avifAlloc(sizeof(avifCodec)); memset(codec, 0, sizeof(struct avifCodec)); - codec->decode = dav1dCodecDecode; + codec->open = dav1dCodecOpen; codec->alphaLimitedRange = dav1dCodecAlphaLimitedRange; codec->getNextImage = dav1dCodecGetNextImage; codec->destroyInternal = dav1dCodecDestroyInternal;
diff --git a/src/read.c b/src/read.c index 06e87ea..ca1a1de 100644 --- a/src/read.c +++ b/src/read.c
@@ -116,12 +116,19 @@ } avifSampleTableTimeToSample; AVIF_ARRAY_DECLARE(avifSampleTableTimeToSampleArray, avifSampleTableTimeToSample, timeToSample); +typedef struct avifSyncSample +{ + uint32_t sampleNumber; +} avifSyncSample; +AVIF_ARRAY_DECLARE(avifSyncSampleArray, avifSyncSample, syncSample); + typedef struct avifSampleTable { avifSampleTableChunkArray chunks; avifSampleTableSampleToChunkArray sampleToChunks; avifSampleTableSampleSizeArray sampleSizes; avifSampleTableTimeToSampleArray timeToSamples; + avifSyncSampleArray syncSamples; } avifSampleTable; static avifSampleTable * avifSampleTableCreate() @@ -132,6 +139,7 @@ avifArrayCreate(&sampleTable->sampleToChunks, sizeof(avifSampleTableSampleToChunk), 16); avifArrayCreate(&sampleTable->sampleSizes, sizeof(avifSampleTableSampleSize), 16); avifArrayCreate(&sampleTable->timeToSamples, sizeof(avifSampleTableTimeToSample), 16); + avifArrayCreate(&sampleTable->syncSamples, sizeof(avifSampleTable), 16); return sampleTable; } @@ -141,6 +149,7 @@ avifArrayDestroy(&sampleTable->sampleToChunks); avifArrayDestroy(&sampleTable->sampleSizes); avifArrayDestroy(&sampleTable->timeToSamples); + avifArrayDestroy(&sampleTable->syncSamples); avifFree(sampleTable); } @@ -177,7 +186,7 @@ { avifCodecDecodeInput * decodeInput = (avifCodecDecodeInput *)avifAlloc(sizeof(avifCodecDecodeInput)); memset(decodeInput, 0, sizeof(avifCodecDecodeInput)); - avifArrayCreate(&decodeInput->samples, sizeof(avifROData), 1); + avifArrayCreate(&decodeInput->samples, sizeof(avifSample), 1); return decodeInput; } @@ -216,9 +225,10 @@ avifSampleTableSampleSize * sampleSize = &sampleTable->sampleSizes.sampleSize[sampleSizeIndex]; - avifROData * rawSample = (avifROData *)avifArrayPushPtr(&decodeInput->samples); - rawSample->data = rawInput->data + sampleOffset; - rawSample->size = sampleSize->size; + avifSample * sample = (avifSample *)avifArrayPushPtr(&decodeInput->samples); + sample->data.data = rawInput->data + sampleOffset; + sample->data.size = sampleSize->size; + sample->sync = AVIF_FALSE; // to potentially be set to true following the outer loop if (sampleOffset > (uint64_t)rawInput->size) { return AVIF_FALSE; @@ -228,6 +238,19 @@ ++sampleSizeIndex; } } + + // Mark appropriate samples as sync + for (uint32_t syncSampleIndex = 0; syncSampleIndex < sampleTable->syncSamples.count; ++syncSampleIndex) { + uint32_t frameIndex = sampleTable->syncSamples.syncSample[syncSampleIndex].sampleNumber - 1; // sampleNumber is 1-based + if (frameIndex < decodeInput->samples.count) { + decodeInput->samples.sample[frameIndex].sync = AVIF_TRUE; + } + } + + // Assume frame 0 is sync, just in case the stss box is absent in the BMFF. (Unnecessary?) + if (decodeInput->samples.count > 0) { + decodeInput->samples.sample[0].sync = AVIF_TRUE; + } return AVIF_TRUE; } @@ -814,6 +837,25 @@ return AVIF_TRUE; } +static avifBool avifParseSyncSampleBox(avifData * data, avifSampleTable * sampleTable, const uint8_t * raw, size_t rawLen) +{ + BEGIN_STREAM(s, raw, rawLen); + (void)data; + + CHECK(avifROStreamReadAndEnforceVersion(&s, 0)); + + uint32_t entryCount; + CHECK(avifROStreamReadU32(&s, &entryCount)); // unsigned int(32) entry_count; + + for (uint32_t i = 0; i < entryCount; ++i) { + uint32_t sampleNumber = 0; + CHECK(avifROStreamReadU32(&s, &sampleNumber)); // unsigned int(32) sample_number; + avifSyncSample * syncSample = (avifSyncSample *)avifArrayPushPtr(&sampleTable->syncSamples); + syncSample->sampleNumber = sampleNumber; + } + return AVIF_TRUE; +} + static avifBool avifParseTimeToSampleBox(avifData * data, avifSampleTable * sampleTable, const uint8_t * raw, size_t rawLen) { BEGIN_STREAM(s, raw, rawLen); @@ -854,6 +896,8 @@ CHECK(avifParseSampleToChunkBox(data, track->sampleTable, avifROStreamCurrent(&s), header.size)); } else if (!memcmp(header.type, "stsz", 4)) { CHECK(avifParseSampleSizeBox(data, track->sampleTable, avifROStreamCurrent(&s), header.size)); + } else if (!memcmp(header.type, "stss", 4)) { + CHECK(avifParseSyncSampleBox(data, track->sampleTable, avifROStreamCurrent(&s), header.size)); } else if (!memcmp(header.type, "stts", 4)) { CHECK(avifParseTimeToSampleBox(data, track->sampleTable, avifROStreamCurrent(&s), header.size)); } @@ -1151,6 +1195,24 @@ return codec; } +static avifResult avifDecoderFlush(avifDecoder * decoder) +{ + avifDataResetCodec(decoder->data); + + decoder->data->codec[AVIF_CODEC_PLANES_COLOR] = avifCodecCreateForDecode(decoder->data->colorInput); + if (!decoder->data->codec[AVIF_CODEC_PLANES_COLOR]->open(decoder->data->codec[AVIF_CODEC_PLANES_COLOR])) { + return AVIF_RESULT_DECODE_COLOR_FAILED; + } + + if (decoder->data->alphaInput) { + decoder->data->codec[AVIF_CODEC_PLANES_ALPHA] = avifCodecCreateForDecode(decoder->data->alphaInput); + if (!decoder->data->codec[AVIF_CODEC_PLANES_ALPHA]->open(decoder->data->codec[AVIF_CODEC_PLANES_ALPHA])) { + return AVIF_RESULT_DECODE_ALPHA_FAILED; + } + } + return AVIF_RESULT_OK; +} + avifResult avifDecoderReset(avifDecoder * decoder) { avifData * data = decoder->data; @@ -1318,12 +1380,14 @@ } data->colorInput = avifCodecDecodeInputCreate(); - avifROData * rawColorInput = (avifROData *)avifArrayPushPtr(&data->colorInput->samples); - memcpy(rawColorInput, &colorOBU, sizeof(avifROData)); + avifSample * colorSample = (avifSample *)avifArrayPushPtr(&data->colorInput->samples); + memcpy(&colorSample->data, &colorOBU, sizeof(avifROData)); + colorSample->sync = AVIF_TRUE; if (alphaOBU.size > 0) { data->alphaInput = avifCodecDecodeInputCreate(); - avifROData * rawAlphaInput = (avifROData *)avifArrayPushPtr(&data->alphaInput->samples); - memcpy(rawAlphaInput, &alphaOBU, sizeof(avifROData)); + avifSample * alphaSample = (avifSample *)avifArrayPushPtr(&data->alphaInput->samples); + memcpy(&alphaSample->data, &alphaOBU, sizeof(avifROData)); + alphaSample->sync = AVIF_TRUE; data->alphaInput->alpha = AVIF_TRUE; } @@ -1343,18 +1407,7 @@ decoder->ioStats.alphaOBUSize = alphaOBU.size; } - data->codec[AVIF_CODEC_PLANES_COLOR] = avifCodecCreateForDecode(data->colorInput); - if (!data->codec[AVIF_CODEC_PLANES_COLOR]->decode(data->codec[AVIF_CODEC_PLANES_COLOR])) { - return AVIF_RESULT_DECODE_COLOR_FAILED; - } - - if (data->alphaInput) { - decoder->data->codec[AVIF_CODEC_PLANES_ALPHA] = avifCodecCreateForDecode(data->alphaInput); - if (!data->codec[AVIF_CODEC_PLANES_ALPHA]->decode(data->codec[AVIF_CODEC_PLANES_ALPHA])) { - return AVIF_RESULT_DECODE_ALPHA_FAILED; - } - } - return AVIF_RESULT_OK; + return avifDecoderFlush(decoder); } avifResult avifDecoderNextImage(avifDecoder * decoder) @@ -1404,7 +1457,10 @@ // Decoding from a track! Provide timing information. decoder->imageTiming.timescale = decoder->timescale; - decoder->imageTiming.ptsInTimescales += decoder->imageTiming.durationInTimescales; + decoder->imageTiming.ptsInTimescales = 0; + for (int imageIndex = 0; imageIndex < decoder->imageIndex; ++imageIndex) { + decoder->imageTiming.ptsInTimescales += avifSampleTableGetImageDelta(decoder->data->sourceSampleTable, imageIndex); + } decoder->imageTiming.durationInTimescales = avifSampleTableGetImageDelta(decoder->data->sourceSampleTable, decoder->imageIndex); if (decoder->imageTiming.timescale > 0) { @@ -1418,6 +1474,60 @@ return AVIF_RESULT_OK; } +avifResult avifDecoderNthImage(avifDecoder * decoder, uint32_t frameIndex) +{ + int requestedIndex = (int)frameIndex; + if (requestedIndex == decoder->imageIndex) { + // We're here already, nothing to do + return AVIF_RESULT_OK; + } + + if (requestedIndex == (decoder->imageIndex + 1)) { + // it's just the next image, nothing special here + return avifDecoderNextImage(decoder); + } + + if (requestedIndex >= decoder->imageCount) { + // Impossible index + return AVIF_RESULT_NO_IMAGES_REMAINING; + } + + // If we get here, a decoder flush is necessary + avifDecoderFlush(decoder); + decoder->imageIndex = ((int)avifDecoderNearestKeyframe(decoder, frameIndex)) - 1; // prepare to read nearest keyframe + for (;;) { + avifResult result = avifDecoderNextImage(decoder); + if (result != AVIF_RESULT_OK) { + return result; + } + + if (requestedIndex == decoder->imageIndex) { + break; + } + }; + return AVIF_RESULT_OK; +} + +avifBool avifDecoderIsKeyframe(avifDecoder * decoder, uint32_t frameIndex) +{ + if (decoder->data->colorInput) { + if (frameIndex < decoder->data->colorInput->samples.count) { + return decoder->data->colorInput->samples.sample[frameIndex].sync; + } + } + return AVIF_FALSE; +} + +uint32_t avifDecoderNearestKeyframe(avifDecoder * decoder, uint32_t frameIndex) +{ + for (; frameIndex != 0; --frameIndex) { + if (avifDecoderIsKeyframe(decoder, frameIndex)) { + break; + } + } + return frameIndex; +} + avifResult avifDecoderRead(avifDecoder * decoder, avifImage * image, avifROData * input) { avifResult result = avifDecoderParse(decoder, input);
diff --git a/tests/aviffuzz.c b/tests/aviffuzz.c index 01c5365..7704682 100644 --- a/tests/aviffuzz.c +++ b/tests/aviffuzz.c
@@ -50,10 +50,12 @@ printf(" * %2.2f seconds, %d images\n", decoder->duration, decoder->imageCount); int frameIndex = 0; while (avifDecoderNextImage(decoder) == AVIF_RESULT_OK) { - printf(" * Decoded frame [%d] [pts %2.2f] [duration %2.2f]: %dx%d\n", + printf(" * Decoded frame [%d] [pts %2.2f] [duration %2.2f] [keyframe:%s nearest:%u]: %dx%d\n", frameIndex, decoder->imageTiming.pts, decoder->imageTiming.duration, + avifDecoderIsKeyframe(decoder, frameIndex) ? "true" : "false", + avifDecoderNearestKeyframe(decoder, frameIndex), decoder->image->width, decoder->image->height); ++frameIndex; @@ -73,6 +75,20 @@ printf("ERROR: Failed to decode image: %s\n", avifResultToString(result)); } +#if 0 + int frameIndex = 25; + if (avifDecoderNthImage(decoder, frameIndex) == AVIF_RESULT_OK) { + printf(" * Decoded frame [%d] [pts %2.2f] [duration %2.2f] [keyframe:%s nearest:%u]: %dx%d\n", + frameIndex, + decoder->imageTiming.pts, + decoder->imageTiming.duration, + avifDecoderIsKeyframe(decoder, frameIndex) ? "true" : "false", + avifDecoderNearestKeyframe(decoder, frameIndex), + decoder->image->width, + decoder->image->height); + } +#endif + avifRWDataFree(&raw); avifDecoderDestroy(decoder); return 0;