commit b729e27fdab29626ca989a63742ccceac5ed8f7b
author Wan-Teh Chang <wtc@google.com> Fri Jul 23 18:51:46 2021 -0700
committer Wan-Teh Chang <wtc@google.com> Fri Oct 01 00:02:34 2021 +0000
tree fe2c1a297a9caabe2ef145c102f4778c091d95d5
parent 750d86390cdeae1d3f08051e5589493716f4763a

Detect chroma subsampling more directly

This is the aom_scale/generic/yv12extend.c and
aom_scale/mips/dspr2/yv12extend_dspr2.c version of
commit bce31a8248b3f339acbf695972384177370c4cb0.

In extend_frame(), detect chroma subsampling more directly by testing
ybf->subsampling_x and ybf->subsampling_y. The original tests do not
work when ybf->y_width or ybf->y_height is equal to 1.

BUG=aomedia:3113

Change-Id: I577a9a4915d22d15950622a1eb130d0859dc76fb
(cherry picked from commit d5779aa619d1bbdbe42f5a7a0241160d5bf26acb)

aom_scale/generic/yv12extend.c

diff --git a/aom_scale/generic/yv12extend.c b/aom_scale/generic/yv12extend.c
index 3d0f4a7..f01def2 100644
--- a/aom_scale/generic/yv12extend.c
+++ b/aom_scale/generic/yv12extend.c
@@ -25,6 +25,7 @@
   assert(src != NULL);
   int i;
   const int linesize = extend_left + extend_right + width;
+  assert(linesize <= src_stride);
 
   /* copy the left and right most columns out */
   uint8_t *src_ptr1 = src;
@@ -66,6 +67,7 @@
                               int extend_bottom, int extend_right) {
   int i;
   const int linesize = extend_left + extend_right + width;
+  assert(linesize <= src_stride);
   uint16_t *src = CONVERT_TO_SHORTPTR(src8);
 
   /* copy the left and right most columns out */
@@ -139,8 +141,8 @@
 
 static void extend_frame(YV12_BUFFER_CONFIG *const ybf, int ext_size,
                          const int num_planes) {
-  const int ss_x = ybf->uv_width < ybf->y_width;
-  const int ss_y = ybf->uv_height < ybf->y_height;
+  const int ss_x = ybf->subsampling_x;
+  const int ss_y = ybf->subsampling_y;
 
   assert(ybf->y_height - ybf->y_crop_height < 16);
   assert(ybf->y_width - ybf->y_crop_width < 16);

aom_scale/mips/dspr2/yv12extend_dspr2.c

diff --git a/aom_scale/mips/dspr2/yv12extend_dspr2.c b/aom_scale/mips/dspr2/yv12extend_dspr2.c
index 869e594..8556e71 100644
--- a/aom_scale/mips/dspr2/yv12extend_dspr2.c
+++ b/aom_scale/mips/dspr2/yv12extend_dspr2.c
@@ -90,6 +90,7 @@
   top_dst = src + src_stride * (-extend_top) - extend_left;
   bot_dst = src + src_stride * (height)-extend_left;
   linesize = extend_left + extend_right + width;
+  assert(linesize <= src_stride);
 
   for (i = 0; i < extend_top; i++) {
     memcpy(top_dst, top_src, linesize);
@@ -105,8 +106,8 @@
 static void extend_frame(YV12_BUFFER_CONFIG *const ybf, int ext_size) {
   const int c_w = ybf->uv_crop_width;
   const int c_h = ybf->uv_crop_height;
-  const int ss_x = ybf->uv_width < ybf->y_width;
-  const int ss_y = ybf->uv_height < ybf->y_height;
+  const int ss_x = ybf->subsampling_x;
+  const int ss_y = ybf->subsampling_y;
   const int c_et = ext_size >> ss_y;
   const int c_el = ext_size >> ss_x;
   const int c_eb = c_et + ybf->uv_height - ybf->uv_crop_height;