aom_img_set_rect: reject a flipped image

Same as libvpx ca46065bc.

Change-Id: Ic5c328f479e840a89766f748467fa55800cd1201

aom/aom_image.h

diff --git a/aom/aom_image.h b/aom/aom_image.h
index 879e3c5..6942249 100644
--- a/aom/aom_image.h
+++ b/aom/aom_image.h
@@ -431,7 +431,8 @@
  *
  * Updates the displayed rectangle (aka viewport) on the image surface to
  * match the specified coordinates and size. Specifically, sets img->d_w,
- * img->d_h, and elements of the img->planes[] array.
+ * img->d_h, and elements of the img->planes[] array. The image must not be
+ * vertically flipped. On failure, the image descriptor is left unchanged.
  *
  * \param[in]    img       Image descriptor
  * \param[in]    x         leftmost column
@@ -440,7 +441,8 @@
  * \param[in]    h         height
  * \param[in]    border    A border that is padded on four sides of the image.
  *
- * \return 0 if the requested rectangle is valid, nonzero (-1) otherwise.
+ * \return 0 if the requested rectangle is valid and the image is not flipped,
+ *         nonzero (-1) otherwise.
  */
 int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y,
                      unsigned int w, unsigned int h, unsigned int border);

aom/src/aom_image.c

diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c
index 69b8f26..c0328cf 100644
--- a/aom/src/aom_image.c
+++ b/aom/src/aom_image.c
@@ -244,6 +244,8 @@
 
 int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y,
                      unsigned int w, unsigned int h, unsigned int border) {
+  if (img->stride[AOM_PLANE_Y] < 0) return -1;
+
   if (x <= UINT_MAX - w && x + w <= img->w && y <= UINT_MAX - h &&
       y + h <= img->h) {
     img->d_w = w;

test/aom_image_test.cc

diff --git a/test/aom_image_test.cc b/test/aom_image_test.cc
index e10c9ad..17614a3 100644
--- a/test/aom_image_test.cc
+++ b/test/aom_image_test.cc
@@ -180,6 +180,46 @@
   aom_img_free(img);
 }
 
+TEST(AomImageTest, AomImgSetRectRejectsFlipped) {
+  static constexpr aom_img_fmt_t kFormats[] = {
+    AOM_IMG_FMT_YV12,   AOM_IMG_FMT_I420,   AOM_IMG_FMT_NV12,
+    AOM_IMG_FMT_I42016, AOM_IMG_FMT_YV1216,
+  };
+
+  for (const aom_img_fmt_t format : kFormats) {
+    SCOPED_TRACE(format);
+    aom_image_t *img = aom_img_alloc(nullptr, format, 4, 4, 1);
+    ASSERT_NE(img, nullptr);
+
+    // An ordinary crop is valid before flipping.
+    ASSERT_EQ(aom_img_set_rect(img, 2, 2, 2, 2, 0), 0);
+    EXPECT_EQ(img->d_w, 2u);
+    EXPECT_EQ(img->d_h, 2u);
+    ASSERT_EQ(aom_img_set_rect(img, 0, 0, 4, 4, 0), 0);
+
+    aom_img_flip(img);
+    const unsigned char *const flipped_planes[] = { img->planes[AOM_PLANE_Y],
+                                                    img->planes[AOM_PLANE_U],
+                                                    img->planes[AOM_PLANE_V] };
+    EXPECT_EQ(aom_img_set_rect(img, 2, 2, 2, 2, 0), -1);
+    EXPECT_EQ(img->d_w, 4u);
+    EXPECT_EQ(img->d_h, 4u);
+    for (int plane = AOM_PLANE_Y; plane <= AOM_PLANE_V; ++plane) {
+      EXPECT_EQ(img->planes[plane], flipped_planes[plane]);
+    }
+
+    // Flipping back permits changing the viewport again.
+    aom_img_flip(img);
+    EXPECT_GT(img->stride[AOM_PLANE_Y], 0);
+    const unsigned char *const unflipped_y_plane = img->planes[AOM_PLANE_Y];
+    EXPECT_EQ(aom_img_set_rect(img, 2, 2, 2, 2, 0), 0);
+    EXPECT_NE(img->planes[AOM_PLANE_Y], unflipped_y_plane);
+    EXPECT_EQ(img->d_w, 2u);
+    EXPECT_EQ(img->d_h, 2u);
+    aom_img_free(img);
+  }
+}
+
 TEST(AomImageTest, AomImgFlipOddHeight) {
   static constexpr aom_img_fmt_t kFormats[] = {
     AOM_IMG_FMT_YV12,   AOM_IMG_FMT_I420,   AOM_IMG_FMT_NV12,