[CFL] Convert cfl_alpha to q3

Alpha's biggest fraction is 1/8, so Q3 does not change the bitstream.

Results on Subset1 (compared to 503aca74 with CfL enabled)

  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |   SSIM | MS SSIM | CIEDE 2000
0.0000 |  0.0000 |  0.0000 |   0.0000 | 0.0000 |  0.0000 |     0.0000

Change-Id: I1fe5b2ace97179d5f950d7406a4f3d391924f89d

av1/common/cfl.c

diff --git a/av1/common/cfl.c b/av1/common/cfl.c
index b5d5997..a2d702f 100644
--- a/av1/common/cfl.c
+++ b/av1/common/cfl.c
@@ -227,16 +227,16 @@
   assert(a <= MAX_NUM_TXB);
 }
 
-static INLINE double cfl_idx_to_alpha(int alpha_idx, CFL_SIGN_TYPE alpha_sign,
-                                      CFL_PRED_TYPE pred_type) {
+static INLINE int cfl_idx_to_alpha(int alpha_idx, CFL_SIGN_TYPE alpha_sign,
+                                   CFL_PRED_TYPE pred_type) {
   const int mag_idx = cfl_alpha_codes[alpha_idx][pred_type];
-  const double abs_alpha = cfl_alpha_mags[mag_idx];
+  const int abs_alpha_q3 = cfl_alpha_mags_q3[mag_idx];
   if (alpha_sign == CFL_SIGN_POS) {
-    return abs_alpha;
+    return abs_alpha_q3;
   } else {
-    assert(abs_alpha != 0.0);
-    assert(cfl_alpha_mags[mag_idx + 1] == -abs_alpha);
-    return -abs_alpha;
+    assert(abs_alpha_q3 != 0);
+    assert(cfl_alpha_mags_q3[mag_idx + 1] == -abs_alpha_q3);
+    return -abs_alpha_q3;
   }
 }
 
@@ -255,10 +255,8 @@
   const uint8_t *y_pix = cfl->y_down_pix;
 
   const int dc_pred = cfl->dc_pred[plane - 1];
-  const double alpha = cfl_idx_to_alpha(
+  const int alpha_q3 = cfl_idx_to_alpha(
       mbmi->cfl_alpha_idx, mbmi->cfl_alpha_signs[plane - 1], plane - 1);
-  // TODO(ltrudeau) Convert alpha to fixed point.
-  const int alpha_q3 = (int)(alpha * 8);
 
   const int avg_row =
       (row << tx_size_wide_log2[0]) >> tx_size_wide_log2[tx_size];

av1/common/cfl.h

diff --git a/av1/common/cfl.h b/av1/common/cfl.h
index 3e61a64..f2df4b2 100644
--- a/av1/common/cfl.h
+++ b/av1/common/cfl.h
@@ -65,9 +65,7 @@
   int mi_row, mi_col;
 } CFL_CTX;
 
-static const double cfl_alpha_mags[CFL_MAGS_SIZE] = {
-  0., 0.125, -0.125, 0.25, -0.25, 0.5, -0.5
-};
+static const int cfl_alpha_mags_q3[CFL_MAGS_SIZE] = { 0, 1, -1, 2, -2, 4, -4 };
 
 static const int cfl_alpha_codes[CFL_ALPHABET_SIZE][CFL_PRED_PLANES] = {
   // barrbrain's simple 1D quant ordered by subset 3 likelihood

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 24cf898..1692005 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -1428,11 +1428,11 @@
                           const int y_averages_q3[MAX_NUM_TXB],
                           const uint8_t *src, int src_stride, int width,
                           int height, TX_SIZE tx_size, int dc_pred,
-                          double alpha, int *dist_neg_out) {
+                          int alpha_q3, int *dist_neg_out) {
   int dist = 0;
   int diff;
 
-  if (alpha == 0.0) {
+  if (alpha_q3 == 0) {
     for (int j = 0; j < height; j++) {
       for (int i = 0; i < width; i++) {
         diff = src[i] - dc_pred;
@@ -1446,8 +1446,6 @@
     return dist;
   }
 
-  // TODO(ltrudeau) Convert alpha to fixed point
-  const int alpha_q3 = (int)(alpha * 8);
   int dist_neg = 0;
   const int tx_height = tx_size_high[tx_size];
   const int tx_width = tx_size_wide[tx_size];
@@ -1543,13 +1541,13 @@
                      width, height, tx_size, dc_pred_v, 0, NULL);
 
   for (int m = 1; m < CFL_MAGS_SIZE; m += 2) {
-    assert(cfl_alpha_mags[m + 1] == -cfl_alpha_mags[m]);
+    assert(cfl_alpha_mags_q3[m + 1] == -cfl_alpha_mags_q3[m]);
     sse[CFL_PRED_U][m] = cfl_alpha_dist(
         y_pix, MAX_SB_SIZE, y_averages_q3, src_u, src_stride_u, width, height,
-        tx_size, dc_pred_u, cfl_alpha_mags[m], &sse[CFL_PRED_U][m + 1]);
+        tx_size, dc_pred_u, cfl_alpha_mags_q3[m], &sse[CFL_PRED_U][m + 1]);
     sse[CFL_PRED_V][m] = cfl_alpha_dist(
         y_pix, MAX_SB_SIZE, y_averages_q3, src_v, src_stride_v, width, height,
-        tx_size, dc_pred_v, cfl_alpha_mags[m], &sse[CFL_PRED_V][m + 1]);
+        tx_size, dc_pred_v, cfl_alpha_mags_q3[m], &sse[CFL_PRED_V][m + 1]);
   }
 
   int dist;