commit 473cf89871aae6aee3234314b3e5b99713e6e2a6
author Hui Su <huisu@google.com> Wed Nov 08 18:14:31 2017 -0800
committer Hui Su <huisu@google.com> Fri Nov 10 01:20:53 2017 +0000
tree 97c42d05c12cb808fb3bea2d9a08b779d6ad2e6d
parent baf09d8a5ec419bc5db4f22256a8c1a8720531a4

Do not allow palette mode for partition sizes larger than 64x64

Tested on screen_content, it has very little impact on compression quality.

Change-Id: I49c7d5e8de8420c0d2af1b327c084a2667cfea40

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 94c6ca4..645ead8 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1175,9 +1175,8 @@
   const MODE_INFO *const above_mi = xd->above_mi;
   const MODE_INFO *const left_mi = xd->left_mi;
   const BLOCK_SIZE bsize = mbmi->sb_type;
+  assert(av1_allow_palette(cm->allow_screen_content_tools, bsize));
   const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
-
-  assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST);
   const int block_palette_idx = bsize - BLOCK_8X8;
 
   if (mbmi->mode == DC_PRED) {

av1/encoder/block.h

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 2dd65d8..065e1a0 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -110,8 +110,8 @@
 } MvLimits;
 
 typedef struct {
-  uint8_t best_palette_color_map[MAX_SB_SQUARE];
-  float kmeans_data_buf[2 * MAX_SB_SQUARE];
+  uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
+  float kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
 } PALETTE_BUFFER;
 
 typedef struct {

av1/encoder/context_tree.c

diff --git a/av1/encoder/context_tree.c b/av1/encoder/context_tree.c
index e88b7d6..1b014dd 100644
--- a/av1/encoder/context_tree.c
+++ b/av1/encoder/context_tree.c
@@ -49,10 +49,12 @@
 #endif
   }
 
-  for (i = 0; i < 2; ++i) {
-    CHECK_MEM_ERROR(
-        cm, ctx->color_index_map[i],
-        aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i])));
+  if (num_pix <= MAX_PALETTE_SQUARE) {
+    for (i = 0; i < 2; ++i) {
+      CHECK_MEM_ERROR(
+          cm, ctx->color_index_map[i],
+          aom_memalign(32, num_pix * sizeof(*ctx->color_index_map[i])));
+    }
   }
 #if CONFIG_MRC_TX
   CHECK_MEM_ERROR(cm, ctx->mrc_mask,

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 862a0ff..53f09d9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3030,7 +3030,7 @@
     mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0;
 #endif  // CONFIG_FILTER_INTRA
 
-    if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return 0;
+    if (rows * cols > MAX_PALETTE_SQUARE) return 0;
 
 #if CONFIG_PALETTE_DELTA_ENCODING
     uint16_t color_cache[2 * PALETTE_MAX_SIZE];
@@ -5118,7 +5118,7 @@
   int plane_block_width, plane_block_height, rows, cols;
   av1_get_block_dimensions(bsize, 1, xd, &plane_block_width,
                            &plane_block_height, &rows, &cols);
-  if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return;
+  if (rows * cols > MAX_PALETTE_SQUARE) return;
 
   mbmi->uv_mode = UV_DC_PRED;
 #if CONFIG_FILTER_INTRA