Downconvert pixels to 8-bit in av1_count_colors_highbd()

This patch modifies the av1_count_colors_highbd() to count the pixels
by down converting them to 8bit domain. This provides consistency of
palette search between lbd and hbd encoding.

STATS_CHANGED for hbd encoding

Change-Id: Ic3777bdbaf68aa5e86cf0bf715febb7e2827a6d4
(cherry picked from commit a39da50ea0f0de0def61f4d08069f198939e52c0)

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 0ee6c6a..c999e71 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -1719,11 +1719,11 @@
 
   for (int r = 0; r + blk_h <= height; r += blk_h) {
     for (int c = 0; c + blk_w <= width; c += blk_w) {
-      int count_buf[1 << 12];  // Maximum (1 << 12) color levels.
+      int count_buf[1 << 8];  // Maximum (1 << 8) bins for hbd path.
       const uint8_t *const this_src = src + r * stride + c;
       const int n_colors =
           use_hbd ? av1_count_colors_highbd(this_src, stride, blk_w, blk_h, bd,
-                                            count_buf)
+                                            NULL, count_buf)
                   : av1_count_colors(this_src, stride, blk_w, blk_h, count_buf);
       if (n_colors > 1 && n_colors <= color_thresh) {
         ++counts_1;

av1/encoder/intra_mode_search.c

diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index 5845b7e..f672ed8 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -177,22 +177,32 @@
 }
 
 int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols,
-                            int bit_depth, int *val_count) {
+                            int bit_depth, int *val_count, int *bin_val_count) {
   assert(bit_depth <= 12);
+  const int max_bin_val = 1 << 8;
   const int max_pix_val = 1 << bit_depth;
   const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
-  memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
+  memset(bin_val_count, 0, max_bin_val * sizeof(val_count[0]));
+  if (val_count != NULL)
+    memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
   for (int r = 0; r < rows; ++r) {
     for (int c = 0; c < cols; ++c) {
-      const int this_val = src[r * stride + c];
-      assert(this_val < max_pix_val);
-      if (this_val >= max_pix_val) return 0;
-      ++val_count[this_val];
+      /*
+       * Down-convert the pixels to 8-bit domain before counting.
+       * This provides consistency of behavior for palette search
+       * between lbd and hbd encodes. This down-converted pixels
+       * are only used for calculating the threshold (n).
+       */
+      const int this_val = ((src[r * stride + c]) >> (bit_depth - 8));
+      assert(this_val < max_bin_val);
+      if (this_val >= max_bin_val) return 0;
+      ++bin_val_count[this_val];
+      if (val_count != NULL) ++val_count[(src[r * stride + c])];
     }
   }
   int n = 0;
-  for (int i = 0; i < max_pix_val; ++i) {
-    if (val_count[i]) ++n;
+  for (int i = 0; i < max_bin_val; ++i) {
+    if (bin_val_count[i]) ++n;
   }
   return n;
 }

av1/encoder/intra_mode_search.h

diff --git a/av1/encoder/intra_mode_search.h b/av1/encoder/intra_mode_search.h
index ab71401..780f2ee 100644
--- a/av1/encoder/intra_mode_search.h
+++ b/av1/encoder/intra_mode_search.h
@@ -235,7 +235,7 @@
 /*! \brief See \ref av1_count_colors(), but for highbd.
  */
 int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols,
-                            int bit_depth, int *val_count);
+                            int bit_depth, int *val_count, int *val_count_8bit);
 
 #ifdef __cplusplus
 }  // extern "C"

av1/encoder/palette.c

diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c
index dd54a8a..b851b0a 100644
--- a/av1/encoder/palette.c
+++ b/av1/encoder/palette.c
@@ -418,11 +418,12 @@
   const int bit_depth = seq_params->bit_depth;
   int unused;
 
-  int count_buf[1 << 12];  // Maximum (1 << 12) color levels.
+  int count_buf[1 << 12];      // Maximum (1 << 12) color levels.
+  int count_buf_8bit[1 << 8];  // Maximum (1 << 8) bins for hbd path.
   int colors;
   if (is_hbd) {
     colors = av1_count_colors_highbd(src, src_stride, rows, cols, bit_depth,
-                                     count_buf);
+                                     count_buf, count_buf_8bit);
   } else {
     colors = av1_count_colors(src, src_stride, rows, cols, count_buf);
   }
@@ -646,13 +647,12 @@
                            &plane_block_height, &rows, &cols);
 
   mbmi->uv_mode = UV_DC_PRED;
-
-  int count_buf[1 << 12];  // Maximum (1 << 12) color levels.
+  int count_buf[1 << 8];  // Maximum (1 << 8) bins for hbd path.
   if (seq_params->use_highbitdepth) {
     colors_u = av1_count_colors_highbd(src_u, src_stride, rows, cols,
-                                       seq_params->bit_depth, count_buf);
+                                       seq_params->bit_depth, NULL, count_buf);
     colors_v = av1_count_colors_highbd(src_v, src_stride, rows, cols,
-                                       seq_params->bit_depth, count_buf);
+                                       seq_params->bit_depth, NULL, count_buf);
   } else {
     colors_u = av1_count_colors(src_u, src_stride, rows, cols, count_buf);
     colors_v = av1_count_colors(src_v, src_stride, rows, cols, count_buf);