commit 5c453db29e1a6645d3bef4e8de0b4ae23e0db4f8
author Luc Trudeau <luc@trud.ca> Tue Jul 04 14:13:35 2017 -0400
committer Luc Trudeau <luc@trud.ca> Thu Jul 06 09:33:11 2017 -0400
tree 1a9f41dfb87746025af197296374565c4a6c00f1
parent 4aee17d7de6535be8f097e7bbd3c9603eec8a6e4

[CFL] clip CFL prediction to avoid overflow

The value predicted using CfL is clipped to avoid going out of the
scope of the uint8. Both overflow and underflow was detected over
Subtset1.

Results on Subset1 (compared to 7e55571 with CfL enabled)

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0019 |  0.0001 |  0.0009 |   0.0047 | 0.0020 |  0.0023 |     0.0012

Change-Id: Ie1190e2286aa90542eaa68b814cc5cfa031acb73

av1/common/cfl.c

diff --git a/av1/common/cfl.c b/av1/common/cfl.c
index 135b308..1ffea03 100644
--- a/av1/common/cfl.c
+++ b/av1/common/cfl.c
@@ -239,7 +239,8 @@
   cfl_load(cfl, row, col, width, height);
   for (int j = 0; j < height; j++) {
     for (int i = 0; i < width; i++) {
-      dst[i] = (uint8_t)(alpha * (y_pix[i] - avg) + dc_pred + 0.5);
+      // TODO(ltrudeau) call clip_pixel_highbd when HBD is enabled.
+      dst[i] = clip_pixel((int)(alpha * (y_pix[i] - avg) + dc_pred + 0.5));
     }
     dst += dst_stride;
     y_pix += MAX_SB_SIZE;

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index e0f4516..f988dee 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -1469,9 +1469,13 @@
         for (int t_i = b_i; t_i < w; t_i++) {
           const double scaled_luma = alpha * (t_y_pix[t_i] - y_average);
           const int uv = t_src[t_i];
-          diff = uv - (int)(scaled_luma + dc_pred_bias);
+
+          // TODO(ltrudeau) add support for HBD.
+          diff = uv - clip_pixel((int)(scaled_luma + dc_pred_bias));
           dist += diff * diff;
-          diff = uv + (int)(scaled_luma - dc_pred_bias);
+
+          // TODO(ltrudeau) add support for HBD.
+          diff = uv - clip_pixel((int)(-scaled_luma + dc_pred_bias));
           dist_neg += diff * diff;
         }
         t_y_pix += y_stride;