Downconvert pixels to 8-bit in av1_count_colors_highbd()
This patch modifies the av1_count_colors_highbd() to count the pixels
by down converting them to 8bit domain. This provides consistency of
palette search between lbd and hbd encoding.
STATS_CHANGED for hbd encoding
Change-Id: Ic3777bdbaf68aa5e86cf0bf715febb7e2827a6d4
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 2be6443..3b50a7d 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -1740,11 +1740,11 @@
for (int r = 0; r + blk_h <= height; r += blk_h) {
for (int c = 0; c + blk_w <= width; c += blk_w) {
- int count_buf[1 << 12]; // Maximum (1 << 12) color levels.
+ int count_buf[1 << 8]; // Maximum (1 << 8) bins for hbd path.
const uint8_t *const this_src = src + r * stride + c;
const int n_colors =
use_hbd ? av1_count_colors_highbd(this_src, stride, blk_w, blk_h, bd,
- count_buf)
+ NULL, count_buf)
: av1_count_colors(this_src, stride, blk_w, blk_h, count_buf);
if (n_colors > 1 && n_colors <= color_thresh) {
++counts_1;
diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index a267943..e372e95 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -177,22 +177,32 @@
}
int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols,
- int bit_depth, int *val_count) {
+ int bit_depth, int *val_count, int *bin_val_count) {
assert(bit_depth <= 12);
+ const int max_bin_val = 1 << 8;
const int max_pix_val = 1 << bit_depth;
const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
- memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
+ memset(bin_val_count, 0, max_bin_val * sizeof(val_count[0]));
+ if (val_count != NULL)
+ memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
for (int r = 0; r < rows; ++r) {
for (int c = 0; c < cols; ++c) {
- const int this_val = src[r * stride + c];
- assert(this_val < max_pix_val);
- if (this_val >= max_pix_val) return 0;
- ++val_count[this_val];
+ /*
+ * Down-convert the pixels to 8-bit domain before counting.
+ * This provides consistency of behavior for palette search
+ * between lbd and hbd encodes. This down-converted pixels
+ * are only used for calculating the threshold (n).
+ */
+ const int this_val = ((src[r * stride + c]) >> (bit_depth - 8));
+ assert(this_val < max_bin_val);
+ if (this_val >= max_bin_val) return 0;
+ ++bin_val_count[this_val];
+ if (val_count != NULL) ++val_count[(src[r * stride + c])];
}
}
int n = 0;
- for (int i = 0; i < max_pix_val; ++i) {
- if (val_count[i]) ++n;
+ for (int i = 0; i < max_bin_val; ++i) {
+ if (bin_val_count[i]) ++n;
}
return n;
}
diff --git a/av1/encoder/intra_mode_search.h b/av1/encoder/intra_mode_search.h
index ab71401..780f2ee 100644
--- a/av1/encoder/intra_mode_search.h
+++ b/av1/encoder/intra_mode_search.h
@@ -235,7 +235,7 @@
/*! \brief See \ref av1_count_colors(), but for highbd.
*/
int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols,
- int bit_depth, int *val_count);
+ int bit_depth, int *val_count, int *val_count_8bit);
#ifdef __cplusplus
} // extern "C"
diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c
index e00e8c1..395380b 100644
--- a/av1/encoder/palette.c
+++ b/av1/encoder/palette.c
@@ -418,11 +418,12 @@
const int bit_depth = seq_params->bit_depth;
int unused;
- int count_buf[1 << 12]; // Maximum (1 << 12) color levels.
+ int count_buf[1 << 12]; // Maximum (1 << 12) color levels.
+ int count_buf_8bit[1 << 8]; // Maximum (1 << 8) bins for hbd path.
int colors;
if (is_hbd) {
colors = av1_count_colors_highbd(src, src_stride, rows, cols, bit_depth,
- count_buf);
+ count_buf, count_buf_8bit);
} else {
colors = av1_count_colors(src, src_stride, rows, cols, count_buf);
}
@@ -646,13 +647,12 @@
&plane_block_height, &rows, &cols);
mbmi->uv_mode = UV_DC_PRED;
-
- int count_buf[1 << 12]; // Maximum (1 << 12) color levels.
+ int count_buf[1 << 8]; // Maximum (1 << 8) bins for hbd path.
if (seq_params->use_highbitdepth) {
colors_u = av1_count_colors_highbd(src_u, src_stride, rows, cols,
- seq_params->bit_depth, count_buf);
+ seq_params->bit_depth, NULL, count_buf);
colors_v = av1_count_colors_highbd(src_v, src_stride, rows, cols,
- seq_params->bit_depth, count_buf);
+ seq_params->bit_depth, NULL, count_buf);
} else {
colors_u = av1_count_colors(src_u, src_stride, rows, cols, count_buf);
colors_v = av1_count_colors(src_v, src_stride, rows, cols, count_buf);
