Prevent integer overflows in src/gainmap.c Prevent integer overflows in multiplications involving width, height, and rowBytes in src/gainmap.c by performing the multiplications in the size_t type. The size_t type is large enough because pixel buffers for the width, height, and rowBytes have been allocated successfully. "Dexter.k" <164054284+rootvector2@users.noreply.github.com> reported an integer overflow in the allocation of the gainMapF buffers in avifRGBImageComputeGainMap() and suggested a fix in https://github.com/AOMediaCodec/libavif/pull/3049.
diff --git a/include/avif/internal.h b/include/avif/internal.h index 2a4f64f..feca658 100644 --- a/include/avif/internal.h +++ b/include/avif/internal.h
@@ -813,7 +813,7 @@ // Uses a histogram, with outliers defined as having at least one empty bucket between them // and the rest of the distribution. Discards at most 0.1% of values. // Removing outliers helps with accuracy/compression. -avifResult avifFindMinMaxWithoutOutliers(const float * gainMapF, int numPixels, float * rangeMin, float * rangeMax); +avifResult avifFindMinMaxWithoutOutliers(const float * gainMapF, size_t numPixels, float * rangeMin, float * rangeMax); avifResult avifGainMapValidateMetadata(const avifGainMap * gainMap, avifDiagnostics * diag);
diff --git a/src/gainmap.c b/src/gainmap.c index b8585bf..40a332f 100644 --- a/src/gainmap.c +++ b/src/gainmap.c
@@ -116,7 +116,7 @@ assert(baseImage->rowBytes == toneMappedImage->rowBytes); assert(baseImage->height == toneMappedImage->height); // Copy the base image. - memcpy(toneMappedImage->pixels, baseImage->pixels, baseImage->rowBytes * baseImage->height); + memcpy(toneMappedImage->pixels, baseImage->pixels, (size_t)baseImage->rowBytes * baseImage->height); goto cleanup; } @@ -285,7 +285,7 @@ // Convert extended SDR (where 1.0 is SDR white) to nits. clli->maxCLL = (uint16_t)AVIF_CLAMP(avifRoundf(rgbMaxLinear * SDR_WHITE_NITS), 0.0f, (float)UINT16_MAX); - const float rgbAverageLinear = rgbSumLinear / (width * height); + const float rgbAverageLinear = rgbSumLinear / ((size_t)width * height); clli->maxPALL = (uint16_t)AVIF_CLAMP(avifRoundf(rgbAverageLinear * SDR_WHITE_NITS), 0.0f, (float)UINT16_MAX); } @@ -355,7 +355,7 @@ return idx * (bucketMax - bucketMin) / numBuckets + bucketMin; } -avifResult avifFindMinMaxWithoutOutliers(const float * gainMapF, int numPixels, float * rangeMin, float * rangeMax) +avifResult avifFindMinMaxWithoutOutliers(const float * gainMapF, size_t numPixels, float * rangeMin, float * rangeMax) { const float bucketSize = 0.01f; // Size of one bucket. Empirical value. const float maxOutliersRatio = 0.001f; // 0.1% @@ -363,7 +363,7 @@ float min = gainMapF[0]; float max = gainMapF[0]; - for (int i = 1; i < numPixels; ++i) { + for (size_t i = 1; i < numPixels; ++i) { min = AVIF_MIN(min, gainMapF[i]); max = AVIF_MAX(max, gainMapF[i]); } @@ -381,7 +381,7 @@ return AVIF_RESULT_OUT_OF_MEMORY; } memset(histogram, 0, sizeof(int) * numBuckets); - for (int i = 0; i < numPixels; ++i) { + for (size_t i = 0; i < numPixels; ++i) { ++(histogram[avifValueToBucketIdx(gainMapF[i], min, max, numBuckets)]); } @@ -541,8 +541,8 @@ const avifBool colorSpacesDiffer = (baseColorPrimaries != altColorPrimaries); avifColorPrimaries gainMapMathPrimaries; AVIF_CHECKRES(avifChooseColorSpaceForGainMapMath(baseColorPrimaries, altColorPrimaries, &gainMapMathPrimaries)); - const int width = baseRgbImage->width; - const int height = baseRgbImage->height; + const uint32_t width = baseRgbImage->width; + const uint32_t height = baseRgbImage->height; avifRGBColorSpaceInfo baseRGBInfo; avifRGBColorSpaceInfo altRGBInfo; @@ -559,10 +559,16 @@ avifResult res = AVIF_RESULT_OK; // --- After this point, the function should exit with 'goto cleanup' to free allocated resources. + const size_t numPixels = (size_t)width * height; + if (numPixels > SIZE_MAX / sizeof(float)) { + res = AVIF_RESULT_INVALID_ARGUMENT; + goto cleanup; + } + const size_t gainMapChannelSize = numPixels * sizeof(float); const avifBool singleChannel = (gainMap->image->yuvFormat == AVIF_PIXEL_FORMAT_YUV400); const int numGainMapChannels = singleChannel ? 1 : 3; for (int c = 0; c < numGainMapChannels; ++c) { - gainMapF[c] = avifAlloc(width * height * sizeof(float)); + gainMapF[c] = avifAlloc(gainMapChannelSize); if (gainMapF[c] == NULL) { res = AVIF_RESULT_OUT_OF_MEMORY; goto cleanup; @@ -608,8 +614,8 @@ // Color convert pure red, pure green and pure blue in turn and see if they result in negative values. float rgba[4] = { 0.0f }; float channelMin[3] = { 0.0f }; - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { + for (uint32_t j = 0; j < height; ++j) { + for (uint32_t i = 0; i < width; ++i) { avifGetRGBAPixel(gainMap->useBaseColorSpace ? altRgbImage : baseRgbImage, i, j, @@ -649,8 +655,8 @@ // Compute raw gain map values. float baseMax = 1.0f; float altMax = 1.0f; - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { + for (uint32_t j = 0; j < height; ++j) { + for (uint32_t i = 0; i < width; ++i) { float baseRGBA[4]; avifGetRGBAPixel(baseRgbImage, i, j, &baseRGBInfo, baseRGBA); float altRGBA[4]; @@ -688,7 +694,7 @@ } const float ratio = (alt + alternateOffset[c]) / (base + baseOffset[c]); const float ratioLog2 = log2f(AVIF_MAX(ratio, kEpsilon)); - gainMapF[c][j * width + i] = ratioLog2; + gainMapF[c][(size_t)j * width + i] = ratioLog2; } } } @@ -707,9 +713,9 @@ // representation. if (alternateHeadroom < baseHeadroom) { for (int c = 0; c < numGainMapChannels; ++c) { - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { - gainMapF[c][j * width + i] *= -1.f; + for (uint32_t j = 0; j < height; ++j) { + for (uint32_t i = 0; i < width; ++i) { + gainMapF[c][(size_t)j * width + i] *= -1.f; } } } @@ -719,7 +725,7 @@ float gainMapMinLog2[3] = { 0.0f, 0.0f, 0.0f }; float gainMapMaxLog2[3] = { 0.0f, 0.0f, 0.0f }; for (int c = 0; c < numGainMapChannels; ++c) { - res = avifFindMinMaxWithoutOutliers(gainMapF[c], width * height, &gainMapMinLog2[c], &gainMapMaxLog2[c]); + res = avifFindMinMaxWithoutOutliers(gainMapF[c], numPixels, &gainMapMinLog2[c], &gainMapMaxLog2[c]); if (res != AVIF_RESULT_OK) { goto cleanup; } @@ -741,22 +747,22 @@ const float range = AVIF_MAX(gainMapMaxLog2[c] - gainMapMinLog2[c], 0.0f); if (range == 0.0f) { - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { + for (uint32_t j = 0; j < height; ++j) { + for (uint32_t i = 0; i < width; ++i) { // If the range is 0, the gain map values will be multiplied by zero when tonemapping so the values // don't matter, but we still need to make sure that gainMapF is in [0,1]. - gainMapF[c][j * width + i] = 0.0f; + gainMapF[c][(size_t)j * width + i] = 0.0f; } } } else { // Remap [min; max] range to [0; 1] const float gainMapGamma = avifUnsignedFractionToFloat(gainMap->gainMapGamma[c]); - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { - float v = gainMapF[c][j * width + i]; + for (uint32_t j = 0; j < height; ++j) { + for (uint32_t i = 0; i < width; ++i) { + float v = gainMapF[c][(size_t)j * width + i]; v = AVIF_CLAMP(v, gainMapMinLog2[c], gainMapMaxLog2[c]); v = powf((v - gainMapMinLog2[c]) / range, gainMapGamma); - gainMapF[c][j * width + i] = AVIF_CLAMP(v, 0.0f, 1.0f); + gainMapF[c][(size_t)j * width + i] = AVIF_CLAMP(v, 0.0f, 1.0f); } } } @@ -785,9 +791,9 @@ avifDiagnosticsPrintf(diag, "Unsupported RGB color space"); return AVIF_RESULT_NOT_IMPLEMENTED; } - for (int j = 0; j < height; ++j) { - for (int i = 0; i < width; ++i) { - const int offset = j * width + i; + for (uint32_t j = 0; j < height; ++j) { + for (uint32_t i = 0; i < width; ++i) { + const size_t offset = (size_t)j * width + i; const float r = gainMapF[0][offset]; const float g = singleChannel ? r : gainMapF[1][offset]; const float b = singleChannel ? r : gainMapF[2][offset];