Fix NaN bypass of AVIF_CLAMP in gain map pixel clamping (#3189)
Apply path (the two sites at lines 156 and 277): when `baseLinear + baseOffset` is zero (e.g., a black pixel with `baseOffset = {0, 1}`), and the gain map log2 value is large enough for `exp2f()` to overflow to +Inf, the multiplication `0.0f * +Inf` evaluates to NaN per IEEE 754. The NaN then passes through `linearToGamma()` and `AVIF_CLAMP()` unmodified, since the macro's ternary comparisons all return false for NaN operands.
A similar NaN path exists in the compute path (line 764): if `base + baseOffset[c]` is near-zero after color space conversion, the ratio at line 695 overflows to +Inf. `AVIF_MAX(ratio, kEpsilon)` only lower-bounds, so +Inf passes through into `gainMapF`. After `avifFindMinMaxWithoutOutliers` picks up +Inf as the channel max, the normalization `(+Inf - min) / +Inf` evaluates to NaN (indeterminate form).
The `fminf`/`fmaxf` fix is intentionally a safety net at the output boundary — it ensures NaN can't reach `avifSetRGBAPixel()` regardless of which upstream path produced it. If you'd prefer a root-cause fix as well (e.g., clamping the ratio or guarding `exp2f` overflow), I'm happy to add that.
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>diff --git a/CHANGELOG.md b/CHANGELOG.md
index 17349e7..7b3b942 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@
* Update LocalAvm.cmake: research-v15.0.0
* Update svt.cmd/svt.sh/LocalSvt.cmake: v4.1.0
* Fix decoding layered image with multiple scaled alpha layers
+* Fix NaN bypass of AVIF_CLAMP in gain map tone mapping (use fminf/fmaxf)
* avifenc: reject mismatched --depth for Y4M input
## [1.4.1] - 2026-03-20
diff --git a/src/gainmap.c b/src/gainmap.c
index 40a332f..f56484d 100644
--- a/src/gainmap.c
+++ b/src/gainmap.c
@@ -7,6 +7,14 @@
#include <math.h>
#include <string.h>
+// NaN-safe clamp to [0, 1]. AVIF_CLAMP passes NaN through because IEEE 754
+// comparisons with NaN always return false. fmaxf/fminf return the non-NaN
+// argument per C99 §7.12.12, so this clamps NaN to 0.
+static float avifNanSafeClamp(float val)
+{
+ return fminf(1.0f, fmaxf(0.0f, val));
+}
+
static void avifGainMapSetEncodingDefaults(avifGainMap * gainMap)
{
for (int i = 0; i < 3; ++i) {
@@ -153,7 +161,7 @@
avifLinearRGBConvertColorSpace(basePixelRGBA, conversionCoeffs);
}
for (int c = 0; c < 3; ++c) {
- basePixelRGBA[c] = AVIF_CLAMP(linearToGamma(basePixelRGBA[c]), 0.0f, 1.0f);
+ basePixelRGBA[c] = avifNanSafeClamp(linearToGamma(basePixelRGBA[c]));
}
}
avifSetRGBAPixel(toneMappedImage, i, j, &toneMappedPixelRGBInfo, basePixelRGBA);
@@ -270,7 +278,12 @@
}
for (int c = 0; c < 3; ++c) {
- toneMappedPixelRGBA[c] = AVIF_CLAMP(linearToGamma(toneMappedPixelRGBA[c]), 0.0f, 1.0f);
+ if (isnan(toneMappedPixelRGBA[c])) {
+ avifDiagnosticsPrintf(diag, "Degenerate gain map parameters produce NaN at pixel (%u, %u)", i, j);
+ res = AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE;
+ goto cleanup;
+ }
+ toneMappedPixelRGBA[c] = avifNanSafeClamp(linearToGamma(toneMappedPixelRGBA[c]));
}
toneMappedPixelRGBA[3] = basePixelRGBA[3]; // Alpha is unaffected by tone mapping.
@@ -762,7 +775,7 @@
float v = gainMapF[c][(size_t)j * width + i];
v = AVIF_CLAMP(v, gainMapMinLog2[c], gainMapMaxLog2[c]);
v = powf((v - gainMapMinLog2[c]) / range, gainMapGamma);
- gainMapF[c][(size_t)j * width + i] = AVIF_CLAMP(v, 0.0f, 1.0f);
+ gainMapF[c][(size_t)j * width + i] = avifNanSafeClamp(v);
}
}
}
diff --git a/tests/gtest/avifgainmaptest.cc b/tests/gtest/avifgainmaptest.cc
index 8abd226..9f6cdd5 100644
--- a/tests/gtest/avifgainmaptest.cc
+++ b/tests/gtest/avifgainmaptest.cc
@@ -1593,6 +1593,94 @@
}
}
+// Verify that applying a gain map with degenerate parameters that produce NaN
+// (0 * Inf from black pixels with baseOffset=0 and large gainMapMax) returns
+// AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE instead of crashing or producing
+// garbage output.
+TEST(GainMapTest, ApplyGainMapNaN) {
+ // 2x2 black base image (sRGB, BT.709).
+ ImagePtr base(avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444));
+ ASSERT_NE(base, nullptr);
+ base->colorPrimaries = AVIF_COLOR_PRIMARIES_SRGB;
+ base->transferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB;
+ base->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709;
+ base->yuvRange = AVIF_RANGE_FULL;
+ ASSERT_EQ(avifImageAllocatePlanes(base.get(), AVIF_PLANES_YUV),
+ AVIF_RESULT_OK)
+ << "Failed to allocate base planes";
+ // Y=0 (black), U=128/V=128 (neutral chroma).
+ memset(base->yuvPlanes[0], 0, (size_t)base->yuvRowBytes[0] * 2);
+ memset(base->yuvPlanes[1], 128, (size_t)base->yuvRowBytes[1] * 2);
+ memset(base->yuvPlanes[2], 128, (size_t)base->yuvRowBytes[2] * 2);
+
+ // 2x2 gain map image — all pixels at maximum (255 -> 1.0 normalized).
+ GainMapPtr gainMap(avifGainMapCreate());
+ ASSERT_NE(gainMap, nullptr);
+ gainMap->image = avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444);
+ ASSERT_NE(gainMap->image, nullptr) << "Failed to create gain map image";
+ gainMap->image->yuvRange = AVIF_RANGE_FULL;
+ gainMap->image->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_IDENTITY;
+ ASSERT_EQ(avifImageAllocatePlanes(gainMap->image, AVIF_PLANES_YUV),
+ AVIF_RESULT_OK)
+ << "Failed to allocate gain map planes";
+ memset(gainMap->image->yuvPlanes[0], 255,
+ (size_t)gainMap->image->yuvRowBytes[0] * 2);
+ memset(gainMap->image->yuvPlanes[1], 255,
+ (size_t)gainMap->image->yuvRowBytes[1] * 2);
+ memset(gainMap->image->yuvPlanes[2], 255,
+ (size_t)gainMap->image->yuvRowBytes[2] * 2);
+
+ // Gain map metadata crafted to trigger NaN:
+ // gainMapMin = 0 -> lerp lower bound
+ // gainMapMax = 1000 -> lerp upper bound
+ // gamma = 1 -> no gamma distortion
+ // baseOffset = 0 -> (baseLinear + 0) = 0 for black pixels
+ // altOffset = 0
+ //
+ // The math: lerp(0, 1000, powf(1.0, 1.0)) = 1000
+ // exp2f(1000 * weight) = +Inf
+ // (0.0 + 0.0) * +Inf = NaN (IEEE 754)
+ for (int c = 0; c < 3; ++c) {
+ gainMap->gainMapMin[c] = {0, 1};
+ gainMap->gainMapMax[c] = {1000, 1};
+ gainMap->gainMapGamma[c] = {1, 1};
+ gainMap->baseOffset[c] = {0, 1};
+ gainMap->alternateOffset[c] = {0, 1};
+ }
+ gainMap->baseHdrHeadroom = {0, 1};
+ gainMap->alternateHdrHeadroom = {6, 1};
+ gainMap->useBaseColorSpace = 1;
+ gainMap->altColorPrimaries = AVIF_COLOR_PRIMARIES_SRGB;
+ gainMap->altTransferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB;
+ gainMap->altMatrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709;
+ gainMap->altYUVRange = AVIF_RANGE_FULL;
+ gainMap->altDepth = 8;
+ gainMap->altPlaneCount = 3;
+
+ // Output tone-mapped image.
+ avifRGBImage toneMap;
+ memset(&toneMap, 0, sizeof(toneMap));
+ toneMap.depth = 8;
+ toneMap.format = AVIF_RGB_FORMAT_RGBA;
+
+ avifContentLightLevelInformationBox clli;
+ memset(&clli, 0, sizeof(clli));
+ avifDiagnostics diag;
+ avifDiagnosticsClearError(&diag);
+
+ // Apply with full HDR headroom (weight = 1.0).
+ // Use LINEAR transfer so NaN propagates through to the clamp check.
+ // (sRGB's gamma function absorbs NaN to 1.0f, hiding the issue.)
+ avifResult result = avifImageApplyGainMap(
+ base.get(), gainMap.get(), 6.0f, AVIF_COLOR_PRIMARIES_SRGB,
+ AVIF_TRANSFER_CHARACTERISTICS_LINEAR, &toneMap, &clli, &diag);
+
+ EXPECT_EQ(result, AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE)
+ << avifResultToString(result) << " (" << diag.error << ")";
+
+ avifRGBImageFreePixels(&toneMap);
+}
+
} // namespace
} // namespace avif
