avifImageYUVAnyToRGBAnySlow: Fix integer overflows
Fix the integer overflows in avifImageYUVAnyToRGBAnySlow() by declaring
a few local variables as size_t or ptrdiff_t.
Declare yuvChannelBytes, yRowBytes, uRowBytes, vRowBytes, aRowBytes as
size_t. These variables are multiplied by index variables of the
uint32_t type. In addition, yuvChannelBytes, uRowBytes, and vRowBytes
are cast to ptrdiff_t or multiplied by -1.
Declare uAdjCol, vAdjCol, uAdjRow, vAdjRow as ptrdiff_t. These variables
may store positive or negative values, and are added to a pointer.
Also remove two pairs of unnecessary parentheses.
Acknowledgments: This pull request replaces the following pull requests
that attempt to fix this issue:
- https://github.com/AOMediaCodec/libavif/pull/3063 by rootvector2
- https://github.com/AOMediaCodec/libavif/pull/3202 by rootvector2
- https://github.com/AOMediaCodec/libavif/pull/3213 by metsw24-max
- https://github.com/AOMediaCodec/libavif/pull/3260 by MXguru1
diff --git a/src/reformat.c b/src/reformat.c
index aff25da..d1a511b 100644
--- a/src/reformat.c
+++ b/src/reformat.c
@@ -659,7 +659,7 @@
float * unormFloatTableY = NULL;
float * unormFloatTableUV = NULL;
AVIF_CHECKERR(avifCreateYUVToRGBLookUpTables(&unormFloatTableY, &unormFloatTableUV, image->depth, state), AVIF_RESULT_OUT_OF_MEMORY);
- const uint32_t yuvChannelBytes = state->yuv.channelBytes;
+ const size_t yuvChannelBytes = state->yuv.channelBytes;
const uint32_t rgbPixelBytes = state->rgb.pixelBytes;
// Aliases for plane data
@@ -667,10 +667,10 @@
const uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U];
const uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V];
const uint8_t * aPlane = image->alphaPlane;
- const uint32_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y];
- const uint32_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U];
- const uint32_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V];
- const uint32_t aRowBytes = image->alphaRowBytes;
+ const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y];
+ const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U];
+ const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V];
+ const size_t aRowBytes = image->alphaRowBytes;
// Various observations and limits
const avifBool yuvHasColor = (uPlane && vPlane && (image->yuvFormat != AVIF_PIXEL_FORMAT_YUV400));
@@ -687,8 +687,8 @@
// uvJ is used only when yuvHasColor is true.
const uint32_t uvJ = yuvHasColor ? (j >> state->yuv.formatInfo.chromaShiftY) : 0;
const uint8_t * ptrY8 = &yPlane[j * yRowBytes];
- const uint8_t * ptrU8 = uPlane ? &uPlane[(uvJ * uRowBytes)] : NULL;
- const uint8_t * ptrV8 = vPlane ? &vPlane[(uvJ * vRowBytes)] : NULL;
+ const uint8_t * ptrU8 = uPlane ? &uPlane[uvJ * uRowBytes] : NULL;
+ const uint8_t * ptrV8 = vPlane ? &vPlane[uvJ * vRowBytes] : NULL;
const uint8_t * ptrA8 = aPlane ? &aPlane[j * aRowBytes] : NULL;
const uint16_t * ptrY16 = (const uint16_t *)ptrY8;
const uint16_t * ptrU16 = (const uint16_t *)ptrU8;
@@ -764,17 +764,17 @@
uint16_t unormU[2][2], unormV[2][2];
// How many bytes to add to a uint8_t pointer index to get to the adjacent (lesser) sample in a given direction
- int uAdjCol, vAdjCol, uAdjRow, vAdjRow;
+ ptrdiff_t uAdjCol, vAdjCol, uAdjRow, vAdjRow;
if ((i == 0) || ((i == (image->width - 1)) && ((i % 2) != 0))) {
uAdjCol = 0;
vAdjCol = 0;
} else {
if ((i % 2) != 0) {
- uAdjCol = yuvChannelBytes;
- vAdjCol = yuvChannelBytes;
+ uAdjCol = (ptrdiff_t)yuvChannelBytes;
+ vAdjCol = (ptrdiff_t)yuvChannelBytes;
} else {
- uAdjCol = -1 * yuvChannelBytes;
- vAdjCol = -1 * yuvChannelBytes;
+ uAdjCol = -1 * (ptrdiff_t)yuvChannelBytes;
+ vAdjCol = -1 * (ptrdiff_t)yuvChannelBytes;
}
}
@@ -786,32 +786,34 @@
vAdjRow = 0;
} else {
if ((j % 2) != 0) {
- uAdjRow = (int)uRowBytes;
- vAdjRow = (int)vRowBytes;
+ uAdjRow = (ptrdiff_t)uRowBytes;
+ vAdjRow = (ptrdiff_t)vRowBytes;
} else {
- uAdjRow = -1 * (int)uRowBytes;
- vAdjRow = -1 * (int)vRowBytes;
+ uAdjRow = -1 * (ptrdiff_t)uRowBytes;
+ vAdjRow = -1 * (ptrdiff_t)vRowBytes;
}
}
+ const uint8_t * pU = &uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes)];
+ const uint8_t * pV = &vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes)];
if (image->depth == 8) {
- unormU[0][0] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes)];
- unormV[0][0] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes)];
- unormU[1][0] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol];
- unormV[1][0] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol];
- unormU[0][1] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjRow];
- unormV[0][1] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjRow];
- unormU[1][1] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol + uAdjRow];
- unormV[1][1] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol + vAdjRow];
+ unormU[0][0] = *pU;
+ unormV[0][0] = *pV;
+ unormU[1][0] = *(pU + uAdjCol);
+ unormV[1][0] = *(pV + vAdjCol);
+ unormU[0][1] = *(pU + uAdjRow);
+ unormV[0][1] = *(pV + vAdjRow);
+ unormU[1][1] = *(pU + uAdjCol + uAdjRow);
+ unormV[1][1] = *(pV + vAdjCol + vAdjRow);
} else {
- unormU[0][0] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes)]);
- unormV[0][0] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes)]);
- unormU[1][0] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol]);
- unormV[1][0] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol]);
- unormU[0][1] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjRow]);
- unormV[0][1] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjRow]);
- unormU[1][1] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol + uAdjRow]);
- unormV[1][1] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol + vAdjRow]);
+ unormU[0][0] = *((const uint16_t *)pU);
+ unormV[0][0] = *((const uint16_t *)pV);
+ unormU[1][0] = *((const uint16_t *)(pU + uAdjCol));
+ unormV[1][0] = *((const uint16_t *)(pV + vAdjCol));
+ unormU[0][1] = *((const uint16_t *)(pU + uAdjRow));
+ unormV[0][1] = *((const uint16_t *)(pV + vAdjRow));
+ unormU[1][1] = *((const uint16_t *)(pU + uAdjCol + uAdjRow));
+ unormV[1][1] = *((const uint16_t *)(pV + vAdjCol + vAdjRow));
// clamp incoming data to protect against bad LUT lookups
for (int bJ = 0; bJ < 2; ++bJ) {
