commit 1ee98b6c1f4c68d32e710ce14489e3647fb7ea51
author Debargha Mukherjee <debargha@google.com> Wed Dec 07 11:24:13 2016 -0800
committer Debargha Mukherjee <debargha@google.com> Wed Dec 07 22:51:52 2016 +0000
tree 79900dd0ddcc425a84512ffb4a7dbc7649580eb8
parent 6e601e956e5b17d88e947950d69a1bc3aad06ecf

Modify boxnum to fix degenerate size videos

Fixes a memory issue and computes values correctly for 1x1 and
1xn or nx1 videos.
This also fixes a test failure in AV1FrameSizeTestsLarge.OneByOneVideo.

Change-Id: Ib5a4efd094763fb6ceb119383c11f776784f58a4

av1/common/restoration.c

diff --git a/av1/common/restoration.c b/av1/common/restoration.c
index 08ab948..b8e6a93 100644
--- a/av1/common/restoration.c
+++ b/av1/common/restoration.c
@@ -428,24 +428,25 @@
 
 static void boxnum(int width, int height, int r, int8_t *num, int num_stride) {
   int i, j;
-  for (i = 0; i <= r; ++i) {
-    for (j = 0; j <= r; ++j) {
-      num[i * num_stride + j] = (r + 1 + i) * (r + 1 + j);
+  for (i = 0; i <= AOMMIN(r, height - 1); ++i) {
+    for (j = 0; j <= AOMMIN(r, width - 1); ++j) {
+      num[i * num_stride + j] =
+          AOMMIN(r + 1 + i, height) * AOMMIN(r + 1 + j, width);
       num[i * num_stride + (width - 1 - j)] = num[i * num_stride + j];
       num[(height - 1 - i) * num_stride + j] = num[i * num_stride + j];
       num[(height - 1 - i) * num_stride + (width - 1 - j)] =
           num[i * num_stride + j];
     }
   }
-  for (j = 0; j <= r; ++j) {
-    const int val = (2 * r + 1) * (r + 1 + j);
+  for (j = 0; j <= AOMMIN(r, width - 1); ++j) {
+    const int val = AOMMIN(2 * r + 1, height) * AOMMIN(r + 1 + j, width);
     for (i = r + 1; i < height - r; ++i) {
       num[i * num_stride + j] = val;
       num[i * num_stride + (width - 1 - j)] = val;
     }
   }
-  for (i = 0; i <= r; ++i) {
-    const int val = (2 * r + 1) * (r + 1 + i);
+  for (i = 0; i <= AOMMIN(r, height - 1); ++i) {
+    const int val = AOMMIN(2 * r + 1, width) * AOMMIN(r + 1 + i, height);
     for (j = r + 1; j < width - r; ++j) {
       num[i * num_stride + j] = val;
       num[(height - 1 - i) * num_stride + j] = val;
@@ -453,7 +454,8 @@
   }
   for (i = r + 1; i < height - r; ++i) {
     for (j = r + 1; j < width - r; ++j) {
-      num[i * num_stride + j] = (2 * r + 1) * (2 * r + 1);
+      num[i * num_stride + j] =
+          AOMMIN(2 * r + 1, height) * AOMMIN(2 * r + 1, width);
     }
   }
 }