commit 42b4be0dfa020fdd747c670ab3e6984b4ebaa271
author Yunqing Wang <yunqingwang@google.com> Mon Dec 17 13:34:08 2018 -0800
committer Yunqing Wang <yunqingwang@google.com> Tue Dec 18 18:22:23 2018 +0000
tree 62ff7e5f163dba8b2d454644390cce2efa3d97fa
parent cc0cc712c544b6cf4d4a9e270215c2e4841e45f5

Correct the border calculation in image2yuvconfig()

The border calculation in image2yuvconfig() doesn't work if the image
width(img->w) is not aligned to 32. This is because y_stride is 32-byte
aligned. For example, if the width is 1200 and we allocate buffer with
a border of 288, then y_stride is 1792.
yv12->border = (yv12->y_stride - img->w) / 2
But the above calculation gives 296, which is not correct.

Change-Id: I0655ff518f263198ee1711de262c6001e6ea4377

av1/av1_iface_common.h

diff --git a/av1/av1_iface_common.h b/av1/av1_iface_common.h
index 713d8c3..5568c89 100644
--- a/av1/av1_iface_common.h
+++ b/av1/av1_iface_common.h
@@ -124,7 +124,12 @@
   } else {
     yv12->flags = 0;
   }
-  yv12->border = (yv12->y_stride - img->w) / 2;
+
+  // Note(yunqing): if img is allocated the same as the frame buffer, y_stride
+  // is 32-byte aligned. Also, handle the cases while allocating img without a
+  // border or stride_align is less than 32.
+  int border = (yv12->y_stride - (int)((img->w + 31) & ~31)) / 2;
+  yv12->border = (border < 0) ? 0 : border;
   yv12->subsampling_x = img->x_chroma_shift;
   yv12->subsampling_y = img->y_chroma_shift;
   return AOM_CODEC_OK;