commit c57924cb9e3322725998b195b212dc154adceb47
author Debargha Mukherjee <debargha@google.com> Thu Nov 03 15:14:27 2016 -0700
committer Debargha Mukherjee <debargha@google.com> Thu Nov 03 16:23:09 2016 -0700
tree 76df240c5e9aed9f7908e3aec996fc8faf3ae3d2
parent d65708a375e4799f50000b0d424e27587d13eb65

Fix bilateral filter asan error for highbitdepth

BUG=webm:1334

Change-Id: I5886eec0a22a8cc056e1bdb493d2faf183816656

av1/common/restoration.c

diff --git a/av1/common/restoration.c b/av1/common/restoration.c
index 8f0a212..f1c4239 100644
--- a/av1/common/restoration.c
+++ b/av1/common/restoration.c
@@ -369,6 +369,7 @@
   const int tile_height = rst->tile_height >> rst->subsampling_y;
   int i, j, subtile_idx;
   int h_start, h_end, v_start, v_end;
+  const int shift = bit_depth - 8;
 
   for (subtile_idx = 0; subtile_idx < BILATERAL_SUBTILES; ++subtile_idx) {
     uint16_t *data_p, *tmpdata_p;
@@ -398,7 +399,7 @@
         for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
           for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
             wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
-                 (int)wr_lut_[data_p2[x] - data_p[j]];
+                 (int)wr_lut_[(data_p2[x] >> shift) - (data_p[j] >> shift)];
             wtsum += (int64_t)wt;
             flsum += (int64_t)wt * data_p2[x];
           }