commit 7bfa86773c55b8ce3450f4421914a045c7801ad8
author Yue Chen <yuec@google.com> Thu May 11 15:03:21 2017 -0700
committer Yue Chen <yuec@google.com> Mon May 15 17:39:52 2017 +0000
tree 3730f851ce3ebee161f09a273086d8a1cd7e988d
parent 3502d08b715bcd20c7cb4f21238adabba2004ff2

Fix filter_intra overflow

(1) clipping output of intra filter right away to stop
unreasonable pixel values from propogating.
(2) fix wrong filter coefficients

Bug=aomedia:418

Change-Id: I2620034ad03b499afb64da6cadb17fc1aa221c44

av1/common/reconintra.c

diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c
index a1ac4dd..8522841 100644
--- a/av1/common/reconintra.c
+++ b/av1/common/reconintra.c
@@ -1216,7 +1216,7 @@
       { 358, 687, -21 },
       { 411, 1083, -470 },
       { 912, 814, -702 },
-      { 883, 902, 761 },
+      { 883, 902, -761 },
   },
 #endif
   {
@@ -1229,7 +1229,7 @@
       { 358, 687, -21 },
       { 411, 1083, -470 },
       { 912, 814, -702 },
-      { 883, 902, 761 },
+      { 883, 902, -761 },
   },
   {
       { 659, 816, -451 },
@@ -1410,13 +1410,11 @@
       ipred = c0 * buffer[r - 1][c] + c1 * buffer[r][c - 1] +
               c2 * buffer[r - 1][c - 1];
       buffer[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
+      buffer[r][c] = clip_pixel(buffer[r][c] + mean) - mean;
     }
 
   for (r = 0; r < bs; ++r) {
-    for (c = 0; c < bs; ++c) {
-      ipred = buffer[r + 1][c + 1] + mean;
-      dst[c] = clip_pixel(ipred);
-    }
+    for (c = 0; c < bs; ++c) dst[c] = clip_pixel(buffer[r + 1][c + 1] + mean);
     dst += stride;
   }
 }
@@ -1455,13 +1453,11 @@
       ipred = c0 * buffer[r - 1][c] + c1 * buffer[r][c - 1] +
               c2 * buffer[r - 1][c - 1] + c3 * buffer[r - 1][c + 1];
       buffer[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
+      buffer[r][c] = clip_pixel(buffer[r][c] + mean) - mean;
     }
 
   for (r = 0; r < bs; ++r) {
-    for (c = 0; c < bs; ++c) {
-      ipred = buffer[r + 1][c + 1] + mean;
-      dst[c] = clip_pixel(ipred);
-    }
+    for (c = 0; c < bs; ++c) dst[c] = clip_pixel(buffer[r + 1][c + 1] + mean);
     dst += stride;
   }
 }
@@ -1630,13 +1626,12 @@
       ipred = c0 * preds[r - 1][c] + c1 * preds[r][c - 1] +
               c2 * preds[r - 1][c - 1];
       preds[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
+      preds[r][c] = clip_pixel_highbd(preds[r][c] + mean, bd) - mean;
     }
 
   for (r = 0; r < bs; ++r) {
-    for (c = 0; c < bs; ++c) {
-      ipred = preds[r + 1][c + 1] + mean;
-      dst[c] = clip_pixel_highbd(ipred, bd);
-    }
+    for (c = 0; c < bs; ++c)
+      dst[c] = clip_pixel_highbd(preds[r + 1][c + 1] + mean, bd);
     dst += stride;
   }
 }
@@ -1676,13 +1671,12 @@
       ipred = c0 * preds[r - 1][c] + c1 * preds[r][c - 1] +
               c2 * preds[r - 1][c - 1] + c3 * preds[r - 1][c + 1];
       preds[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
+      preds[r][c] = clip_pixel_highbd(preds[r][c] + mean, bd) - mean;
     }
 
   for (r = 0; r < bs; ++r) {
-    for (c = 0; c < bs; ++c) {
-      ipred = preds[r + 1][c + 1] + mean;
-      dst[c] = clip_pixel_highbd(ipred, bd);
-    }
+    for (c = 0; c < bs; ++c)
+      dst[c] = clip_pixel_highbd(preds[r + 1][c + 1] + mean, bd);
     dst += stride;
   }
 }