Add borders to w first, before all manipulations. This allows the border samples to be multiplied by bps / bit_depth for non-planar formats. Change-Id: I4c7e2d4fd6e21049963bf21ee9e55ee6683dde30
diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c index 09b6dd4..f8d4c78 100644 --- a/aom/src/aom_image.c +++ b/aom/src/aom_image.c
@@ -118,8 +118,8 @@ const unsigned int h = align_image_dimension(d_h, ycs, size_align); assert(d_h <= h); - uint64_t s = (fmt & AOM_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / bit_depth; - s = s + 2 * border; + uint64_t s = (uint64_t)w + 2 * border; + s = (fmt & AOM_IMG_FMT_PLANAR) ? s : s * bps / bit_depth; s = s * bit_depth / 8; s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1); if (s > INT_MAX) goto fail;