Make palette work correctly with chroma sub8x8 blocks.
The problem was that some functions were using scale_chroma_bsize()
function to turn sub-8x8 'bsize' to 8x8 'bsize', and then the modified
'bsize' was being passed to rd_pick_intra_sbuv_mode() for example.
In such cases, we cannot rely on the 'bsize' value passed to the
function; instead, we need to look at the original mbmi->sb_type
directly.
Also:
- Added created a common function can_use_palette() to refactor this
logic into one place.
- Added more asserts to easily catch such coding errors in future.
BUG=aomedia:688
Change-Id: I2e9f20c8c5fbc4b3ff41b703a91a02758c3c632f
diff --git a/av1/decoder/detokenize.c b/av1/decoder/detokenize.c
index 3aefc9f..89d9ef3 100644
--- a/av1/decoder/detokenize.c
+++ b/av1/decoder/detokenize.c
@@ -231,6 +231,7 @@
plane ? xd->tile_ctx->palette_uv_color_index_cdf
: xd->tile_ctx->palette_y_color_index_cdf;
int plane_block_width, plane_block_height, rows, cols;
+ assert(mbmi->sb_type >= BLOCK_8X8);
av1_get_block_dimensions(mbmi->sb_type, plane, xd, &plane_block_width,
&plane_block_height, &rows, &cols);
assert(plane == 0 || plane == 1);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 6736787..ba45253 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3107,6 +3107,7 @@
MODE_INFO *const mic = xd->mi[0];
MB_MODE_INFO *const mbmi = &mic->mbmi;
assert(!is_inter_block(mbmi));
+ assert(bsize >= BLOCK_8X8);
int this_rate, colors, n;
const int src_stride = x->plane[0].src.stride;
const uint8_t *const src = x->plane[0].src.buf;
@@ -4269,6 +4270,15 @@
#endif // CONFIG_HIGHBITDEPTH
#endif // CONFIG_EXT_INTRA
+#if CONFIG_PALETTE
+// Returns true if palette can be used for this block.
+static int can_use_palette(const AV1_COMP *const cpi,
+ const MB_MODE_INFO *const mbmi) {
+ return cpi->common.allow_screen_content_tools && mbmi->sb_type >= BLOCK_8X8 &&
+ mbmi->sb_type <= BLOCK_LARGEST;
+}
+#endif // CONFIG_PALETTE
+
// This function is used only for intra_only frames
static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
int *rate, int *rate_tokenonly,
@@ -4303,8 +4313,7 @@
? x->palette_buffer->best_palette_color_map
: NULL;
int palette_y_mode_ctx = 0;
- const int try_palette = cpi->common.allow_screen_content_tools &&
- bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST;
+ const int try_palette = can_use_palette(cpi, mbmi);
#endif // CONFIG_PALETTE
const MODE_INFO *above_mi = xd->above_mi;
const MODE_INFO *left_mi = xd->left_mi;
@@ -5520,6 +5529,7 @@
assert(!is_inter_block(mbmi));
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
const BLOCK_SIZE bsize = mbmi->sb_type;
+ assert(bsize >= BLOCK_8X8);
int this_rate;
int64_t this_rd;
int colors_u, colors_v, colors;
@@ -6038,8 +6048,7 @@
#endif // CONFIG_PVQ
#if CONFIG_PALETTE
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
- const int try_palette = cpi->common.allow_screen_content_tools &&
- bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST;
+ const int try_palette = can_use_palette(cpi, mbmi);
#endif // CONFIG_PALETTE
for (int mode_idx = 0; mode_idx < UV_INTRA_MODES; ++mode_idx) {
@@ -10065,6 +10074,7 @@
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
const BLOCK_SIZE bsize = mbmi->sb_type;
+ assert(bsize >= BLOCK_8X8);
int src_stride = x->plane[1].src.stride;
const uint8_t *const src_u = x->plane[1].src.buf;
const uint8_t *const src_v = x->plane[2].src.buf;
@@ -10134,8 +10144,7 @@
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
#if CONFIG_PALETTE
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
- const int try_palette = cpi->common.allow_screen_content_tools &&
- bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST;
+ const int try_palette = can_use_palette(cpi, mbmi);
#endif // CONFIG_PALETTE
int rate2 = 0, rate_y = INT_MAX, skippable = 0, rate_uv, rate_dummy, i;
int dc_mode_index;
@@ -10305,8 +10314,7 @@
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
#if CONFIG_PALETTE
- const int try_palette =
- cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8;
+ const int try_palette = can_use_palette(cpi, mbmi);
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
#endif // CONFIG_PALETTE
MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
@@ -12278,7 +12286,8 @@
best_mode_skippable);
#if CONFIG_PALETTE
- if (cm->allow_screen_content_tools && pmi->palette_size[1] > 0) {
+ if (pmi->palette_size[1] > 0) {
+ assert(try_palette);
restore_uv_color_map(cpi, x);
}
#endif // CONFIG_PALETTE
