commit 473cf89871aae6aee3234314b3e5b99713e6e2a6
author Hui Su <huisu@google.com> Wed Nov 08 18:14:31 2017 -0800
committer Hui Su <huisu@google.com> Fri Nov 10 01:20:53 2017 +0000
tree 97c42d05c12cb808fb3bea2d9a08b779d6ad2e6d
parent baf09d8a5ec419bc5db4f22256a8c1a8720531a4

Do not allow palette mode for partition sizes larger than 64x64

Tested on screen_content, it has very little impact on compression quality.

Change-Id: I49c7d5e8de8420c0d2af1b327c084a2667cfea40

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 862a0ff..53f09d9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3030,7 +3030,7 @@
     mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0;
 #endif  // CONFIG_FILTER_INTRA
 
-    if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return 0;
+    if (rows * cols > MAX_PALETTE_SQUARE) return 0;
 
 #if CONFIG_PALETTE_DELTA_ENCODING
     uint16_t color_cache[2 * PALETTE_MAX_SIZE];
@@ -5118,7 +5118,7 @@
   int plane_block_width, plane_block_height, rows, cols;
   av1_get_block_dimensions(bsize, 1, xd, &plane_block_width,
                            &plane_block_height, &rows, &cols);
-  if (rows * cols > PALETTE_MAX_BLOCK_SIZE) return;
+  if (rows * cols > MAX_PALETTE_SQUARE) return;
 
   mbmi->uv_mode = UV_DC_PRED;
 #if CONFIG_FILTER_INTRA