Do not allow palette mode for partition sizes larger than 64x64 Tested on screen_content, it has very little impact on compression quality. Change-Id: I49c7d5e8de8420c0d2af1b327c084a2667cfea40
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 5d12740..1644057 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -1389,7 +1389,7 @@ static INLINE int av1_allow_palette(int allow_screen_content_tools, BLOCK_SIZE sb_type) { return allow_screen_content_tools && sb_type >= BLOCK_8X8 && - sb_type <= BLOCK_LARGEST; + sb_type <= BLOCK_64X64; } // Returns sub-sampled dimensions of the given block.
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c index 705d265..5ac7e15 100644 --- a/av1/common/entropymode.c +++ b/av1/common/entropymode.c
@@ -1195,11 +1195,6 @@ { AOM_CDF7(23039, 27333, 30220, 30708, 31070, 31826) }, { AOM_CDF7(13696, 18911, 23620, 25371, 29821, 31617) }, { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) }, -#if CONFIG_EXT_PARTITION - { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) }, - { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) }, - { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) }, -#endif }; const aom_cdf_prob default_palette_uv_size_cdf[PALETTE_BLOCK_SIZES][CDF_SIZE( @@ -1214,26 +1209,24 @@ { AOM_CDF7(8576, 13585, 17632, 20884, 23948, 27152) }, { AOM_CDF7(15360, 24200, 26978, 30846, 31409, 32545) }, { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) }, -#if CONFIG_EXT_PARTITION - { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) }, - { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) }, - { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) }, -#endif }; // When palette mode is enabled, following probability tables indicate the // probabilities to code the "is_palette" bit (i.e. the bit that indicates // if this block uses palette mode or DC_PRED mode). -const aom_prob av1_default_palette_y_mode_prob - [PALETTE_BLOCK_SIZES][PALETTE_Y_MODE_CONTEXTS] = { - { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, - { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, - { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, - { 240, 180, 100 }, -#if CONFIG_EXT_PARTITION - { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 }, -#endif // CONFIG_EXT_PARTITION - }; +const aom_prob av1_default_palette_y_mode_prob[PALETTE_BLOCK_SIZES] + [PALETTE_Y_MODE_CONTEXTS] = { + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + { 240, 180, 100 }, + }; const aom_prob av1_default_palette_uv_mode_prob[PALETTE_UV_MODE_CONTEXTS] = { 253, 229 @@ -1273,24 +1266,12 @@ { { AOM_CDF2(128 * 240) }, { AOM_CDF2(128 * 180) }, { AOM_CDF2(128 * 100) } }, -#if CONFIG_EXT_PARTITION - { { AOM_CDF2(128 * 240) }, - { AOM_CDF2(128 * 180) }, - { AOM_CDF2(128 * 100) } }, - { { AOM_CDF2(128 * 240) }, - { AOM_CDF2(128 * 180) }, - { AOM_CDF2(128 * 100) } }, - { { AOM_CDF2(128 * 240) }, - { AOM_CDF2(128 * 180) }, - { AOM_CDF2(128 * 100) } }, -#endif // CONFIG_EXT_PARTITION }; const aom_cdf_prob default_palette_uv_mode_cdf[PALETTE_UV_MODE_CONTEXTS][CDF_SIZE(2)] = { { AOM_CDF2(128 * 253) }, { AOM_CDF2(128 * 229) } }; - #endif const aom_cdf_prob default_palette_y_color_index_cdf
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h index f263bda..310e58a 100644 --- a/av1/common/entropymode.h +++ b/av1/common/entropymode.h
@@ -38,14 +38,6 @@ // a value from 0 to 4 using 'palette_color_index_context_lookup' table. #define PALETTE_COLOR_INDEX_CONTEXTS 5 -// Maximum number of colors in a palette. -#define PALETTE_MAX_SIZE 8 -// Minimum number of colors in a palette. -#define PALETTE_MIN_SIZE 2 - -// Palette mode is available for block sizes >= 8x8. -#define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1) - // Palette Y mode context for a block is determined by number of neighboring // blocks (top and/or left) using a palette for Y plane. So, possible Y mode' // context values are: @@ -60,8 +52,6 @@ // 1 if this block uses palette for Y plane (i.e. Y palette size > 0). #define PALETTE_UV_MODE_CONTEXTS 2 -#define PALETTE_MAX_BLOCK_SIZE (64 * 64) - #if CONFIG_KF_CTX #define KF_MODE_CONTEXTS 5 #endif
diff --git a/av1/common/enums.h b/av1/common/enums.h index fe4ce59..8da1854 100644 --- a/av1/common/enums.h +++ b/av1/common/enums.h
@@ -72,6 +72,17 @@ #define MI_SIZE_64X64 (64 >> MI_SIZE_LOG2) +#define MIN_PALETTE_BSIZE BLOCK_8X8 +#define MAX_PALETTE_BSIZE BLOCK_64X64 +#define MAX_PALETTE_SQUARE (64 * 64) +// Maximum number of colors in a palette. +#define PALETTE_MAX_SIZE 8 +// Minimum number of colors in a palette. +#define PALETTE_MIN_SIZE 2 +// Palette mode is available for block sizes between MIN_PALETTE_BSIZE and +// MAX_PALETTE_BSIZE. +#define PALETTE_BLOCK_SIZES (MAX_PALETTE_BSIZE - MIN_PALETTE_BSIZE + 1) + #if CONFIG_LOOPFILTER_LEVEL // 4 frame filter levels: y plane vertical, y plane horizontal, // u plane, and v plane
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 914c1a8..4826de2 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -840,9 +840,8 @@ const MODE_INFO *const above_mi = xd->above_mi; const MODE_INFO *const left_mi = xd->left_mi; const BLOCK_SIZE bsize = mbmi->sb_type; + assert(av1_allow_palette(cm->allow_screen_content_tools, bsize)); PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; - - assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST); const int block_palette_idx = bsize - BLOCK_8X8; int modev;
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h index dcd0caf..1fe106c 100644 --- a/av1/decoder/decoder.h +++ b/av1/decoder/decoder.h
@@ -44,7 +44,7 @@ CFL_CTX cfl; #endif DECLARE_ALIGNED(16, FRAME_CONTEXT, tctx); - DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_PALETTE_SQUARE]); #if CONFIG_MRC_TX DECLARE_ALIGNED(16, uint8_t, mrc_mask[MAX_SB_SQUARE]); #endif // CONFIG_MRC_TX @@ -61,7 +61,7 @@ CFL_CTX cfl; #endif FRAME_CONTEXT tctx; - DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_SB_SQUARE]); + DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_PALETTE_SQUARE]); #if CONFIG_MRC_TX DECLARE_ALIGNED(16, uint8_t, mrc_mask[MAX_SB_SQUARE]); #endif // CONFIG_MRC_TX
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 94c6ca4..645ead8 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -1175,9 +1175,8 @@ const MODE_INFO *const above_mi = xd->above_mi; const MODE_INFO *const left_mi = xd->left_mi; const BLOCK_SIZE bsize = mbmi->sb_type; + assert(av1_allow_palette(cm->allow_screen_content_tools, bsize)); const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; - - assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST); const int block_palette_idx = bsize - BLOCK_8X8; if (mbmi->mode == DC_PRED) {
diff --git a/av1/encoder/block.h b/av1/encoder/block.h index 2dd65d8..065e1a0 100644 --- a/av1/encoder/block.h +++ b/av1/encoder/block.h
@@ -110,8 +110,8 @@ } MvLimits; typedef struct { - uint8_t best_palette_color_map[MAX_SB_SQUARE]; - float kmeans_data_buf[2 * MAX_SB_SQUARE]; + uint8_t best_palette_color_map[MAX_PALETTE_SQUARE]; + float kmeans_data_buf[2 * MAX_PALETTE_SQUARE]; } PALETTE_BUFFER; typedef struct {
diff --git a/av1/encoder/context_tree.c b/av1/encoder/context_tree.c index e88b7d6..1b014dd 100644 --- a/av1/encoder/context_tree.c +++ b/av1/encoder/context_tree.c
@@ -49,10 +49,12 @@ #endif } - for (i = 0; i < 2; ++i) { - CHECK_MEM_ERROR( - cm, ctx->color_index_map[i], - aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i]))); + if (num_pix <= MAX_PALETTE_SQUARE) { + for (i = 0; i < 2; ++i) { + CHECK_MEM_ERROR( + cm, ctx->color_index_map[i], + aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i]))); + } } #if CONFIG_MRC_TX CHECK_MEM_ERROR(cm, ctx->mrc_mask,
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 862a0ff..53f09d9 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -3030,7 +3030,7 @@ mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0; #endif // CONFIG_FILTER_INTRA - if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return 0; + if (rows * cols > MAX_PALETTE_SQUARE) return 0; #if CONFIG_PALETTE_DELTA_ENCODING uint16_t color_cache[2 * PALETTE_MAX_SIZE]; @@ -5118,7 +5118,7 @@ int plane_block_width, plane_block_height, rows, cols; av1_get_block_dimensions(bsize, 1, xd, &plane_block_width, &plane_block_height, &rows, &cols); - if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return; + if (rows * cols > MAX_PALETTE_SQUARE) return; mbmi->uv_mode = UV_DC_PRED; #if CONFIG_FILTER_INTRA