commit ec1369152abc738b37aca7bb8a859ab268273249
author Wan-Teh Chang <wtc@google.com> Tue Jul 13 19:12:40 2021 +0000
committer Wan-Teh Chang <wtc@google.com> Tue Jul 13 20:42:32 2021 +0000
tree e724b5eaec39a757defe8513027dbc0bd12faf1d
parent 84fbe62d0042a2708fbcbfdcc00c473979f89712

Revert "Change the evaluation order of luma palette_size in speed 0"

This reverts commit 39a924880d3efc909d5b26f2e0d94b57ecc3317a.

Reason for revert: GCC -Wstack-usage=100000 warning:

aom/av1/encoder/palette.c: In function ‘av1_rd_pick_palette_intra_sby’:
aom/av1/encoder/palette.c:457:6: warning: stack usage might be 100752 bytes [-Wstack-usage=]
  457 | void av1_rd_pick_palette_intra_sby(
      |      ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~

BUG=aomedia:3100
BUG=aomedia:3096

Change-Id: I71717fb4d72e53b9fad089590308ca472a6d6fbc

av1/encoder/palette.c

diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c
index 1b0bf62..04d2390 100644
--- a/av1/encoder/palette.c
+++ b/av1/encoder/palette.c
@@ -600,35 +600,6 @@
             distortion, skippable, beat_best_rd, ctx, best_blk_skip,
             tx_type_map, color_map, rows * cols);
       }
-    } else if (cpi->sf.intra_sf.prune_palette_search_level == 0) {
-      const int max_n = AOMMIN(colors, PALETTE_MAX_SIZE),
-                min_n = PALETTE_MIN_SIZE;
-      // Perform top color palette search in ascending order.
-      perform_top_color_palette_search(
-          cpi, x, mbmi, bsize, dc_mode_cost, data, top_colors, min_n, max_n + 1,
-          1, /*do_header_rd_based_gating=*/false, &unused, color_cache, n_cache,
-          best_mbmi, best_palette_color_map, best_rd, rate, rate_tokenonly,
-          distortion, skippable, beat_best_rd, ctx, best_blk_skip, tx_type_map);
-      // K-means clustering.
-      if (colors == PALETTE_MIN_SIZE) {
-        // Special case: These colors automatically become the centroids.
-        assert(colors == 2);
-        centroids[0] = lower_bound;
-        centroids[1] = upper_bound;
-        palette_rd_y(cpi, x, mbmi, bsize, dc_mode_cost, data, centroids, colors,
-                     color_cache, n_cache, /*do_header_rd_based_gating=*/false,
-                     best_mbmi, best_palette_color_map, best_rd, rate,
-                     rate_tokenonly, distortion, skippable, beat_best_rd, ctx,
-                     best_blk_skip, tx_type_map, NULL, NULL);
-      } else {
-        // Perform k-means palette search in ascending order.
-        perform_k_means_palette_search(
-            cpi, x, mbmi, bsize, dc_mode_cost, data, lower_bound, upper_bound,
-            min_n, max_n + 1, 1, /*do_header_rd_based_gating=*/false, &unused,
-            color_cache, n_cache, best_mbmi, best_palette_color_map, best_rd,
-            rate, rate_tokenonly, distortion, skippable, beat_best_rd, ctx,
-            best_blk_skip, tx_type_map, color_map, rows * cols);
-      }
     } else {
       const int max_n = AOMMIN(colors, PALETTE_MAX_SIZE),
                 min_n = PALETTE_MIN_SIZE;