avifImageYUVAnyToRGBAnySlow: Fix integer overflows Fix the integer overflows in avifImageYUVAnyToRGBAnySlow() by declaring a few local variables as size_t or ptrdiff_t. Declare yuvChannelBytes, yRowBytes, uRowBytes, vRowBytes, aRowBytes as size_t. These variables are multiplied by index variables of the uint32_t type. In addition, yuvChannelBytes, uRowBytes, and vRowBytes are cast to ptrdiff_t or multiplied by -1. Declare uAdjCol, vAdjCol, uAdjRow, vAdjRow as ptrdiff_t. These variables may store positive or negative values, and are added to a pointer. Also remove two pairs of unnecessary parentheses. Acknowledgments: This pull request replaces the following pull requests that attempt to fix this issue: - https://github.com/AOMediaCodec/libavif/pull/3063 by rootvector2 - https://github.com/AOMediaCodec/libavif/pull/3202 by rootvector2 - https://github.com/AOMediaCodec/libavif/pull/3213 by metsw24-max - https://github.com/AOMediaCodec/libavif/pull/3260 by MXguru1
diff --git a/src/reformat.c b/src/reformat.c index aff25da..d1a511b 100644 --- a/src/reformat.c +++ b/src/reformat.c
@@ -659,7 +659,7 @@ float * unormFloatTableY = NULL; float * unormFloatTableUV = NULL; AVIF_CHECKERR(avifCreateYUVToRGBLookUpTables(&unormFloatTableY, &unormFloatTableUV, image->depth, state), AVIF_RESULT_OUT_OF_MEMORY); - const uint32_t yuvChannelBytes = state->yuv.channelBytes; + const size_t yuvChannelBytes = state->yuv.channelBytes; const uint32_t rgbPixelBytes = state->rgb.pixelBytes; // Aliases for plane data @@ -667,10 +667,10 @@ const uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U]; const uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V]; const uint8_t * aPlane = image->alphaPlane; - const uint32_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; - const uint32_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; - const uint32_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; - const uint32_t aRowBytes = image->alphaRowBytes; + const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; + const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; + const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; + const size_t aRowBytes = image->alphaRowBytes; // Various observations and limits const avifBool yuvHasColor = (uPlane && vPlane && (image->yuvFormat != AVIF_PIXEL_FORMAT_YUV400)); @@ -687,8 +687,8 @@ // uvJ is used only when yuvHasColor is true. const uint32_t uvJ = yuvHasColor ? (j >> state->yuv.formatInfo.chromaShiftY) : 0; const uint8_t * ptrY8 = &yPlane[j * yRowBytes]; - const uint8_t * ptrU8 = uPlane ? &uPlane[(uvJ * uRowBytes)] : NULL; - const uint8_t * ptrV8 = vPlane ? &vPlane[(uvJ * vRowBytes)] : NULL; + const uint8_t * ptrU8 = uPlane ? &uPlane[uvJ * uRowBytes] : NULL; + const uint8_t * ptrV8 = vPlane ? &vPlane[uvJ * vRowBytes] : NULL; const uint8_t * ptrA8 = aPlane ? &aPlane[j * aRowBytes] : NULL; const uint16_t * ptrY16 = (const uint16_t *)ptrY8; const uint16_t * ptrU16 = (const uint16_t *)ptrU8; @@ -764,17 +764,17 @@ uint16_t unormU[2][2], unormV[2][2]; // How many bytes to add to a uint8_t pointer index to get to the adjacent (lesser) sample in a given direction - int uAdjCol, vAdjCol, uAdjRow, vAdjRow; + ptrdiff_t uAdjCol, vAdjCol, uAdjRow, vAdjRow; if ((i == 0) || ((i == (image->width - 1)) && ((i % 2) != 0))) { uAdjCol = 0; vAdjCol = 0; } else { if ((i % 2) != 0) { - uAdjCol = yuvChannelBytes; - vAdjCol = yuvChannelBytes; + uAdjCol = (ptrdiff_t)yuvChannelBytes; + vAdjCol = (ptrdiff_t)yuvChannelBytes; } else { - uAdjCol = -1 * yuvChannelBytes; - vAdjCol = -1 * yuvChannelBytes; + uAdjCol = -1 * (ptrdiff_t)yuvChannelBytes; + vAdjCol = -1 * (ptrdiff_t)yuvChannelBytes; } } @@ -786,32 +786,34 @@ vAdjRow = 0; } else { if ((j % 2) != 0) { - uAdjRow = (int)uRowBytes; - vAdjRow = (int)vRowBytes; + uAdjRow = (ptrdiff_t)uRowBytes; + vAdjRow = (ptrdiff_t)vRowBytes; } else { - uAdjRow = -1 * (int)uRowBytes; - vAdjRow = -1 * (int)vRowBytes; + uAdjRow = -1 * (ptrdiff_t)uRowBytes; + vAdjRow = -1 * (ptrdiff_t)vRowBytes; } } + const uint8_t * pU = &uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes)]; + const uint8_t * pV = &vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes)]; if (image->depth == 8) { - unormU[0][0] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes)]; - unormV[0][0] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes)]; - unormU[1][0] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol]; - unormV[1][0] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol]; - unormU[0][1] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjRow]; - unormV[0][1] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjRow]; - unormU[1][1] = uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol + uAdjRow]; - unormV[1][1] = vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol + vAdjRow]; + unormU[0][0] = *pU; + unormV[0][0] = *pV; + unormU[1][0] = *(pU + uAdjCol); + unormV[1][0] = *(pV + vAdjCol); + unormU[0][1] = *(pU + uAdjRow); + unormV[0][1] = *(pV + vAdjRow); + unormU[1][1] = *(pU + uAdjCol + uAdjRow); + unormV[1][1] = *(pV + vAdjCol + vAdjRow); } else { - unormU[0][0] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes)]); - unormV[0][0] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes)]); - unormU[1][0] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol]); - unormV[1][0] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol]); - unormU[0][1] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjRow]); - unormV[0][1] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjRow]); - unormU[1][1] = *((const uint16_t *)&uPlane[(uvJ * uRowBytes) + (uvI * yuvChannelBytes) + uAdjCol + uAdjRow]); - unormV[1][1] = *((const uint16_t *)&vPlane[(uvJ * vRowBytes) + (uvI * yuvChannelBytes) + vAdjCol + vAdjRow]); + unormU[0][0] = *((const uint16_t *)pU); + unormV[0][0] = *((const uint16_t *)pV); + unormU[1][0] = *((const uint16_t *)(pU + uAdjCol)); + unormV[1][0] = *((const uint16_t *)(pV + vAdjCol)); + unormU[0][1] = *((const uint16_t *)(pU + uAdjRow)); + unormV[0][1] = *((const uint16_t *)(pV + vAdjRow)); + unormU[1][1] = *((const uint16_t *)(pU + uAdjCol + uAdjRow)); + unormV[1][1] = *((const uint16_t *)(pV + vAdjCol + vAdjRow)); // clamp incoming data to protect against bad LUT lookups for (int bJ = 0; bJ < 2; ++bJ) {