commit b889a7fa7c8b0842a5d5b675e1aa419e205c5aef
author Wan-Teh Chang <wtc@google.com> Fri Jun 28 16:35:44 2019 -0700
committer Wan-Teh Chang <wtc@google.com> Wed Jul 10 23:36:05 2019 +0000
tree 90b3f454f8e25cff6fba9299ff523e2ac9535688
parent 9d05576c16e671e5665a563c3127b5636ecc9488

Do not account for high bit depth twice.

When calculating the stride (for packed formats) and alloc_size, we
should not account for high bit depth twice.

bps accounts for high bit depth (implicitly).

stride_in_bytes also accounts for high bit depth.

So any value calculated by multiplying bps and stride_in_bytes ends up
accounting for high bit depth twice. A solution is to divide bps not by
a fixed 8 bits but by a variable (bit_depth) that is either 8 bits or 16
bits depending on the bit depth.

Change-Id: I603db355eb7ed923da6d55634d5c247095c08356

aom/src/aom_image.c

diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c
index 723ff83..9407fac 100644
--- a/aom/src/aom_image.c
+++ b/aom/src/aom_image.c
@@ -33,7 +33,10 @@
     unsigned int buf_align, unsigned int stride_align, unsigned int size_align,
     unsigned int border, unsigned char *img_data,
     aom_alloc_img_data_cb_fn_t alloc_cb, void *cb_priv) {
-  unsigned int h, w, s, xcs, ycs, bps;
+  /* NOTE: In this function, bit_depth is either 8 or 16 (if
+   * AOM_IMG_FMT_HIGHBITDEPTH is set), never 10 or 12.
+   */
+  unsigned int h, w, s, xcs, ycs, bps, bit_depth;
   unsigned int stride_in_bytes;
 
   /* Treat align==0 like align==1 */
@@ -69,6 +72,8 @@
     default: bps = 16; break;
   }
 
+  bit_depth = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 16 : 8;
+
   /* Get chroma shift values for this format */
   switch (fmt) {
     case AOM_IMG_FMT_I420:
@@ -96,9 +101,9 @@
   w = align_image_dimension(d_w, xcs, size_align);
   h = align_image_dimension(d_h, ycs, size_align);
 
-  s = (fmt & AOM_IMG_FMT_PLANAR) ? w : bps * w / 8;
+  s = (fmt & AOM_IMG_FMT_PLANAR) ? w : bps * w / bit_depth;
   s = (s + 2 * border + stride_align - 1) & ~(stride_align - 1);
-  stride_in_bytes = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? s * 2 : s;
+  stride_in_bytes = s * bit_depth / 8;
 
   /* Allocate the new image */
   if (!img) {
@@ -116,7 +121,7 @@
   if (!img_data) {
     const uint64_t alloc_size =
         (fmt & AOM_IMG_FMT_PLANAR)
-            ? (uint64_t)(h + 2 * border) * stride_in_bytes * bps / 8
+            ? (uint64_t)(h + 2 * border) * stride_in_bytes * bps / bit_depth
             : (uint64_t)(h + 2 * border) * stride_in_bytes;
 
     if (alloc_size != (size_t)alloc_size) goto fail;
@@ -138,7 +143,7 @@
   if (!img->img_data) goto fail;
 
   img->fmt = fmt;
-  img->bit_depth = (fmt & AOM_IMG_FMT_HIGHBITDEPTH) ? 16 : 8;
+  img->bit_depth = bit_depth;
   // aligned width and aligned height
   img->w = w;
   img->h = h;