commit 736114f44bb52639d545d46ce1900ea1e32de46c
author Jim Bankoski <jimbankoski@google.com> Thu Sep 05 15:53:31 2013 -0700
committer Jim Bankoski <jimbankoski@google.com> Thu Sep 05 15:53:31 2013 -0700
tree 3f987667a145d598c4555c00175b686dc01613a6
parent 170be56a74e72b4d53c6d1de193d2df24a204862

fix loop filter setup_mask could reach out of bounds issue

Change-Id: Ic8446c4f26b6782a6dc482c19ea73c77646df418

vp9/common/vp9_loopfilter.c

diff --git a/vp9/common/vp9_loopfilter.c b/vp9/common/vp9_loopfilter.c
index df806ac..0d883ab 100644
--- a/vp9/common/vp9_loopfilter.c
+++ b/vp9/common/vp9_loopfilter.c
@@ -603,11 +603,15 @@
     case BLOCK_64X32:
       build_masks(lfi_n, mip, 0, 0, lfm);
       mip2 = mip + mode_info_stride * 4;
+      if (4 >= max_rows)
+        break;
       build_masks(lfi_n, mip2 , 32, 8, lfm);
       break;
     case BLOCK_32X64:
       build_masks(lfi_n, mip, 0, 0, lfm);
       mip2 = mip + 4;
+      if (4 >= max_cols)
+        break;
       build_masks(lfi_n, mip2, 4, 2, lfm);
       break;
     default:
@@ -624,11 +628,15 @@
             break;
           case BLOCK_32X16:
             build_masks(lfi_n, mip, shift_y, shift_uv, lfm);
+            if (mi_32_row_offset + 2 >= max_rows)
+              continue;
             mip2 = mip + mode_info_stride * 2;
             build_masks(lfi_n, mip2, shift_y + 16, shift_uv + 4, lfm);
             break;
           case BLOCK_16X32:
             build_masks(lfi_n, mip, shift_y, shift_uv, lfm);
+            if (mi_32_col_offset + 2 >= max_cols)
+              continue;
             mip2 = mip + 2;
             build_masks(lfi_n, mip2, shift_y + 2, shift_uv + 1, lfm);
             break;
@@ -650,11 +658,15 @@
                   break;
                 case BLOCK_16X8:
                   build_masks(lfi_n, mip, shift_y, shift_uv, lfm);
+                  if (mi_16_row_offset + 1 >= max_rows)
+                    continue;
                   mip2 = mip + mode_info_stride;
                   build_y_mask(lfi_n, mip2, shift_y+8, lfm);
                   break;
                 case BLOCK_8X16:
                   build_masks(lfi_n, mip, shift_y, shift_uv, lfm);
+                  if (mi_16_col_offset +1 >= max_cols)
+                    continue;
                   mip2 = mip + 1;
                   build_y_mask(lfi_n, mip2, shift_y+1, lfm);
                   break;