Cast (height - 1) to ptrdiff_t in aom_img_flip() aom_img_flip() casts (height - 1) to `signed` before multiplying it with stride. Change the `signed` casts to `ptrdiff_t` casts to avoid overflows on 64-bit systems. Since the img->planes[] buffers have been allocated successfully, the (height - 1) * stride products can be represented in the `ptrdiff_t` type, so the `ptrdiff_t` casts won't truncate (height - 1). Change-Id: I3ab4b887e832eb2e663d58ccdf9ec98e8375d26c
diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c index 881429a..69b8f26 100644 --- a/aom/src/aom_image.c +++ b/aom/src/aom_image.c
@@ -12,6 +12,7 @@ #include <assert.h> #include <limits.h> #include <stdbool.h> +#include <stddef.h> #include <stdlib.h> #include <string.h> @@ -296,27 +297,21 @@ const unsigned int chroma_height = (img->d_h + img->y_chroma_shift) >> img->y_chroma_shift; - /* Note: In the pointer adjustment calculations, we want the rhs to be - * promoted to a signed type. Section 6.3.1.8 of the ISO C99 standard - * indicates that if the first operand of the multiplication is unsigned, the - * stride will be promoted to unsigned, causing errors when the lhs is a - * larger type than the rhs. - */ if (img->planes[AOM_PLANE_Y]) { img->planes[AOM_PLANE_Y] += - (signed)(img->d_h - 1) * img->stride[AOM_PLANE_Y]; + (ptrdiff_t)(img->d_h - 1) * img->stride[AOM_PLANE_Y]; } img->stride[AOM_PLANE_Y] = -img->stride[AOM_PLANE_Y]; if (img->planes[AOM_PLANE_U]) { img->planes[AOM_PLANE_U] += - (signed)(chroma_height - 1) * img->stride[AOM_PLANE_U]; + (ptrdiff_t)(chroma_height - 1) * img->stride[AOM_PLANE_U]; } img->stride[AOM_PLANE_U] = -img->stride[AOM_PLANE_U]; if (img->planes[AOM_PLANE_V]) { img->planes[AOM_PLANE_V] += - (signed)(chroma_height - 1) * img->stride[AOM_PLANE_V]; + (ptrdiff_t)(chroma_height - 1) * img->stride[AOM_PLANE_V]; } img->stride[AOM_PLANE_V] = -img->stride[AOM_PLANE_V]; }