commit 2dffbafc910846896f1fb77b4a47173ba26f781e
author Cheng Chen <chengchen@google.com> Fri Sep 21 11:59:06 2018 -0700
committer Cheng Chen <chengchen@google.com> Fri Sep 21 22:58:05 2018 +0000
tree 9aa3133ee79cc8368aa55740d0a8a754551ebef4
parent 3b16e00988cd933ff346b2fa6e5819902fb04f27

Fix a fuzzer bug for loop filter bitmask

Horizontal loop filter takes two edges to filter together.
This CL fix the bug where the next edge is out of bound of current
64x64 block.

BUG=oss-fuzz:10574

Change-Id: I3fbe27ec1c33a1bb9728413b3b883539653b7fdb

av1/common/av1_loopfilter.c

diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c
index ff24a3e..0b2446f 100644
--- a/av1/common/av1_loopfilter.c
+++ b/av1/common/av1_loopfilter.c
@@ -1319,12 +1319,16 @@
   int count;
   const int step = 1 << subsampling;
   const unsigned int two_block_mask = subsampling ? 5 : 3;
+  int offset = 0;
 
   for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= step * count) {
     const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;
-    // Next block's thresholds.
-    const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + step);
-    (void)lfin;
+    // Next block's thresholds, when it is within current 64x64 block.
+    // If it is out of bound, its mask is zero, and it points to current edge's
+    // filter parameters, instead of next edge's.
+    int next_edge = step;
+    if (offset + next_edge >= MI_SIZE_64X64) next_edge = 0;
+    const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + next_edge);
 
     count = 1;
     if (mask & 1) {
@@ -1385,6 +1389,7 @@
     mask_16x16 >>= step * count;
     mask_8x8 >>= step * count;
     mask_4x4 >>= step * count;
+    offset += step * count;
   }
 }
 
@@ -1396,12 +1401,16 @@
   int count;
   const int step = 1 << subsampling;
   const unsigned int two_block_mask = subsampling ? 5 : 3;
+  int offset = 0;
 
   for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= step * count) {
     const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;
-    // Next block's thresholds.
-    const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + step);
-    (void)lfin;
+    // Next block's thresholds, when it is within current 64x64 block.
+    // If it is out of bound, its mask is zero, and it points to current edge's
+    // filter parameters, instead of next edge's.
+    int next_edge = step;
+    if (offset + next_edge >= MI_SIZE_64X64) next_edge = 0;
+    const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + next_edge);
 
     count = 1;
     if (mask & 1) {
@@ -1461,6 +1470,7 @@
     mask_16x16 >>= step * count;
     mask_8x8 >>= step * count;
     mask_4x4 >>= step * count;
+    offset += step * count;
   }
 }