Do not allow palette mode for partition sizes larger than 64x64
Tested on screen_content, it has very little impact on compression quality.
Change-Id: I49c7d5e8de8420c0d2af1b327c084a2667cfea40
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 5d12740..1644057 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1389,7 +1389,7 @@
static INLINE int av1_allow_palette(int allow_screen_content_tools,
BLOCK_SIZE sb_type) {
return allow_screen_content_tools && sb_type >= BLOCK_8X8 &&
- sb_type <= BLOCK_LARGEST;
+ sb_type <= BLOCK_64X64;
}
// Returns sub-sampled dimensions of the given block.
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 705d265..5ac7e15 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1195,11 +1195,6 @@
{ AOM_CDF7(23039, 27333, 30220, 30708, 31070, 31826) },
{ AOM_CDF7(13696, 18911, 23620, 25371, 29821, 31617) },
{ AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) },
-#if CONFIG_EXT_PARTITION
- { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) },
- { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) },
- { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) },
-#endif
};
const aom_cdf_prob default_palette_uv_size_cdf[PALETTE_BLOCK_SIZES][CDF_SIZE(
@@ -1214,26 +1209,24 @@
{ AOM_CDF7(8576, 13585, 17632, 20884, 23948, 27152) },
{ AOM_CDF7(15360, 24200, 26978, 30846, 31409, 32545) },
{ AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) },
-#if CONFIG_EXT_PARTITION
- { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) },
- { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) },
- { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) },
-#endif
};
// When palette mode is enabled, following probability tables indicate the
// probabilities to code the "is_palette" bit (i.e. the bit that indicates
// if this block uses palette mode or DC_PRED mode).
-const aom_prob av1_default_palette_y_mode_prob
- [PALETTE_BLOCK_SIZES][PALETTE_Y_MODE_CONTEXTS] = {
- { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 },
- { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 },
- { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 },
- { 240, 180, 100 },
-#if CONFIG_EXT_PARTITION
- { 240, 180, 100 }, { 240, 180, 100 }, { 240, 180, 100 },
-#endif // CONFIG_EXT_PARTITION
- };
+const aom_prob av1_default_palette_y_mode_prob[PALETTE_BLOCK_SIZES]
+ [PALETTE_Y_MODE_CONTEXTS] = {
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ { 240, 180, 100 },
+ };
const aom_prob av1_default_palette_uv_mode_prob[PALETTE_UV_MODE_CONTEXTS] = {
253, 229
@@ -1273,24 +1266,12 @@
{ { AOM_CDF2(128 * 240) },
{ AOM_CDF2(128 * 180) },
{ AOM_CDF2(128 * 100) } },
-#if CONFIG_EXT_PARTITION
- { { AOM_CDF2(128 * 240) },
- { AOM_CDF2(128 * 180) },
- { AOM_CDF2(128 * 100) } },
- { { AOM_CDF2(128 * 240) },
- { AOM_CDF2(128 * 180) },
- { AOM_CDF2(128 * 100) } },
- { { AOM_CDF2(128 * 240) },
- { AOM_CDF2(128 * 180) },
- { AOM_CDF2(128 * 100) } },
-#endif // CONFIG_EXT_PARTITION
};
const aom_cdf_prob
default_palette_uv_mode_cdf[PALETTE_UV_MODE_CONTEXTS][CDF_SIZE(2)] = {
{ AOM_CDF2(128 * 253) }, { AOM_CDF2(128 * 229) }
};
-
#endif
const aom_cdf_prob default_palette_y_color_index_cdf
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index f263bda..310e58a 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -38,14 +38,6 @@
// a value from 0 to 4 using 'palette_color_index_context_lookup' table.
#define PALETTE_COLOR_INDEX_CONTEXTS 5
-// Maximum number of colors in a palette.
-#define PALETTE_MAX_SIZE 8
-// Minimum number of colors in a palette.
-#define PALETTE_MIN_SIZE 2
-
-// Palette mode is available for block sizes >= 8x8.
-#define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1)
-
// Palette Y mode context for a block is determined by number of neighboring
// blocks (top and/or left) using a palette for Y plane. So, possible Y mode'
// context values are:
@@ -60,8 +52,6 @@
// 1 if this block uses palette for Y plane (i.e. Y palette size > 0).
#define PALETTE_UV_MODE_CONTEXTS 2
-#define PALETTE_MAX_BLOCK_SIZE (64 * 64)
-
#if CONFIG_KF_CTX
#define KF_MODE_CONTEXTS 5
#endif
diff --git a/av1/common/enums.h b/av1/common/enums.h
index fe4ce59..8da1854 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -72,6 +72,17 @@
#define MI_SIZE_64X64 (64 >> MI_SIZE_LOG2)
+#define MIN_PALETTE_BSIZE BLOCK_8X8
+#define MAX_PALETTE_BSIZE BLOCK_64X64
+#define MAX_PALETTE_SQUARE (64 * 64)
+// Maximum number of colors in a palette.
+#define PALETTE_MAX_SIZE 8
+// Minimum number of colors in a palette.
+#define PALETTE_MIN_SIZE 2
+// Palette mode is available for block sizes between MIN_PALETTE_BSIZE and
+// MAX_PALETTE_BSIZE.
+#define PALETTE_BLOCK_SIZES (MAX_PALETTE_BSIZE - MIN_PALETTE_BSIZE + 1)
+
#if CONFIG_LOOPFILTER_LEVEL
// 4 frame filter levels: y plane vertical, y plane horizontal,
// u plane, and v plane
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 914c1a8..4826de2 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -840,9 +840,8 @@
const MODE_INFO *const above_mi = xd->above_mi;
const MODE_INFO *const left_mi = xd->left_mi;
const BLOCK_SIZE bsize = mbmi->sb_type;
+ assert(av1_allow_palette(cm->allow_screen_content_tools, bsize));
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
-
- assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST);
const int block_palette_idx = bsize - BLOCK_8X8;
int modev;
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h
index dcd0caf..1fe106c 100644
--- a/av1/decoder/decoder.h
+++ b/av1/decoder/decoder.h
@@ -44,7 +44,7 @@
CFL_CTX cfl;
#endif
DECLARE_ALIGNED(16, FRAME_CONTEXT, tctx);
- DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_PALETTE_SQUARE]);
#if CONFIG_MRC_TX
DECLARE_ALIGNED(16, uint8_t, mrc_mask[MAX_SB_SQUARE]);
#endif // CONFIG_MRC_TX
@@ -61,7 +61,7 @@
CFL_CTX cfl;
#endif
FRAME_CONTEXT tctx;
- DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_SB_SQUARE]);
+ DECLARE_ALIGNED(16, uint8_t, color_index_map[2][MAX_PALETTE_SQUARE]);
#if CONFIG_MRC_TX
DECLARE_ALIGNED(16, uint8_t, mrc_mask[MAX_SB_SQUARE]);
#endif // CONFIG_MRC_TX
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 94c6ca4..645ead8 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1175,9 +1175,8 @@
const MODE_INFO *const above_mi = xd->above_mi;
const MODE_INFO *const left_mi = xd->left_mi;
const BLOCK_SIZE bsize = mbmi->sb_type;
+ assert(av1_allow_palette(cm->allow_screen_content_tools, bsize));
const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
-
- assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST);
const int block_palette_idx = bsize - BLOCK_8X8;
if (mbmi->mode == DC_PRED) {
diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 2dd65d8..065e1a0 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -110,8 +110,8 @@
} MvLimits;
typedef struct {
- uint8_t best_palette_color_map[MAX_SB_SQUARE];
- float kmeans_data_buf[2 * MAX_SB_SQUARE];
+ uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
+ float kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
} PALETTE_BUFFER;
typedef struct {
diff --git a/av1/encoder/context_tree.c b/av1/encoder/context_tree.c
index e88b7d6..1b014dd 100644
--- a/av1/encoder/context_tree.c
+++ b/av1/encoder/context_tree.c
@@ -49,10 +49,12 @@
#endif
}
- for (i = 0; i < 2; ++i) {
- CHECK_MEM_ERROR(
- cm, ctx->color_index_map[i],
- aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i])));
+ if (num_pix <= MAX_PALETTE_SQUARE) {
+ for (i = 0; i < 2; ++i) {
+ CHECK_MEM_ERROR(
+ cm, ctx->color_index_map[i],
+ aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i])));
+ }
}
#if CONFIG_MRC_TX
CHECK_MEM_ERROR(cm, ctx->mrc_mask,
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 862a0ff..53f09d9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3030,7 +3030,7 @@
mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0;
#endif // CONFIG_FILTER_INTRA
- if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return 0;
+ if (rows * cols > MAX_PALETTE_SQUARE) return 0;
#if CONFIG_PALETTE_DELTA_ENCODING
uint16_t color_cache[2 * PALETTE_MAX_SIZE];
@@ -5118,7 +5118,7 @@
int plane_block_width, plane_block_height, rows, cols;
av1_get_block_dimensions(bsize, 1, xd, &plane_block_width,
&plane_block_height, &rows, &cols);
- if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return;
+ if (rows * cols > MAX_PALETTE_SQUARE) return;
mbmi->uv_mode = UV_DC_PRED;
#if CONFIG_FILTER_INTRA
