commit cbb8be769df5dfbf53b70a11981bf53e3c40b418
author hui su <huisu@google.com> Tue Mar 29 16:28:28 2016 -0700
committer hui su <huisu@google.com> Wed Mar 30 14:39:44 2016 -0700
tree aaf4f2c98155811a7ba81b01a4151a1e37b98c93
parent 91707ac79e9fda0b32817ea143e30e64521fbe66

Set block size upper bound for Palette mode

Avoid buffer overflow in case of such new experiments as
128 x 128 superblock size.

Change-Id: Ib775f3925a85fc87227c0ddd9b6a6110a12ef196

vp10/common/entropymode.h

diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h
index 8219dc5..b14fbfd 100644
--- a/vp10/common/entropymode.h
+++ b/vp10/common/entropymode.h
@@ -34,6 +34,7 @@
 #define PALETTE_MAX_SIZE 8
 #define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1)
 #define PALETTE_Y_MODE_CONTEXTS 3
+#define PALETTE_MAX_BLOCK_SIZE (64 * 64)
 
 struct VP10Common;
 

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 328e70c..481a17e 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1823,6 +1823,9 @@
     mbmi->ext_intra_mode_info.use_ext_intra_mode[0] = 0;
 #endif  // CONFIG_EXT_INTRA
 
+    if (rows * cols > PALETTE_MAX_BLOCK_SIZE)
+      return 0;
+
     for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors;
         n >= 2; --n) {
       for (i = 0; i < n; ++i)
@@ -3552,6 +3555,9 @@
   const uint8_t *const src_u = x->plane[1].src.buf;
   const uint8_t *const src_v = x->plane[2].src.buf;
 
+  if (rows * cols > PALETTE_MAX_BLOCK_SIZE)
+    return;
+
 #if CONFIG_EXT_INTRA
   mbmi->ext_intra_mode_info.use_ext_intra_mode[1] = 0;
 #endif  // CONFIG_EXT_INTRA