commit ec2b87871e895fa133dfc38950577a0e1108882f
author Jayasanker J <jayasanker.j@ittiam.com> Wed Jul 14 17:45:01 2021 +0530
committer Wan-Teh Chang <wtc@google.com> Thu Jul 15 01:24:58 2021 +0000
tree ae7175a0606c2af6e9a4a3cb3c0fb8303a2f42f3
parent 2c9a623905881b189711f0e217f02b0fe5435b21

Fix GCC -Wstack-usage warning in av1_rd_pick_palette_intra_sby

This patch reduces the stack memory usage in the function
av1_rd_pick_palette_intra_sby by declaring pre_indices in
RENAME(av1_k_means) with the number of pixels in the largest
possible block size for palette mode.

BUG=aomedia:3100
BUG=aomedia:3096

Change-Id: Ic4d44c7706b69e7a333d918acc0a9065c3f99e8a

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index a4994d0..b2e72d2 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -41,6 +41,10 @@
 
 #define DEFAULT_INTER_TX_TYPE DCT_DCT
 
+#define MAX_PALETTE_BLOCK_WIDTH 64
+
+#define MAX_PALETTE_BLOCK_HEIGHT 64
+
 /*!\cond */
 
 // DIFFWTD_MASK_TYPES should not surpass 1 << MAX_DIFFWTD_MASK_BITS
@@ -1501,8 +1505,10 @@
 static INLINE int av1_allow_palette(int allow_screen_content_tools,
                                     BLOCK_SIZE sb_type) {
   assert(sb_type < BLOCK_SIZES_ALL);
-  return allow_screen_content_tools && block_size_wide[sb_type] <= 64 &&
-         block_size_high[sb_type] <= 64 && sb_type >= BLOCK_8X8;
+  return allow_screen_content_tools &&
+         block_size_wide[sb_type] <= MAX_PALETTE_BLOCK_WIDTH &&
+         block_size_high[sb_type] <= MAX_PALETTE_BLOCK_HEIGHT &&
+         sb_type >= BLOCK_8X8;
 }
 
 // Returns sub-sampled dimensions of the given block.

av1/encoder/k_means_template.h

diff --git a/av1/encoder/k_means_template.h b/av1/encoder/k_means_template.h
index 84c52a2..e794caf 100644
--- a/av1/encoder/k_means_template.h
+++ b/av1/encoder/k_means_template.h
@@ -13,6 +13,7 @@
 #include <stdint.h>
 #include <string.h>
 
+#include "av1/common/blockd.h"
 #include "av1/encoder/palette.h"
 #include "av1/encoder/random.h"
 
@@ -93,7 +94,9 @@
 void RENAME(av1_k_means)(const int *data, int *centroids, uint8_t *indices,
                          int n, int k, int max_itr) {
   int pre_centroids[2 * PALETTE_MAX_SIZE];
-  uint8_t pre_indices[MAX_SB_SQUARE];
+  uint8_t pre_indices[MAX_PALETTE_BLOCK_WIDTH * MAX_PALETTE_BLOCK_HEIGHT];
+
+  assert(n <= MAX_PALETTE_BLOCK_WIDTH * MAX_PALETTE_BLOCK_HEIGHT);
 
 #if AV1_K_MEANS_DIM - 2
   av1_calc_indices_dim1(data, centroids, indices, n, k);