rtc: Adjust color_thresh_palette for nonrd keyframe

For keyframes with hybrid_intra_pickmode = 0:
reduce color_thresh_palette down, to exit palette search
early when it's likely not the best mode.

bdrate on rtc_screen for speed 11, all-intra with CQ mode:
avg_psnr/ovr_psnr/ssim:-6.3, -5.3, -7.1
Average Instruction count reduction ~17%.

For the default IPPP CBR mode:
avg_psnr/ovr_psnr/ssim: 0.153, 0.619, 0.427
Some loss on screen_recording_crd clip, that will
be looked into.

Change-Id: I2fd8acb674d09c34629f08f6e593ec97a807d9d4

av1/encoder/block.h

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 1baf3f9..9bee0b8 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -1325,6 +1325,9 @@
   //! Coding block distortion value for uv/color, minimum over the inter modes.
   int64_t min_dist_inter_uv;
 
+  //! Threshold on the number of colors for testing palette mode.
+  int color_palette_thresh;
+
   //! The buffer used by search_tx_type() to swap dqcoeff in macroblockd_plane
   // so we can keep dqcoeff of the best tx_type.
   tran_low_t *dqcoeff_buf;

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 07382eb..4c178b1 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1211,6 +1211,7 @@
     x->sb_me_partition = 0;
     x->sb_me_mv.as_int = 0;
     x->sb_force_fixed_part = 1;
+    x->color_palette_thresh = 64;
 
     if (cpi->oxcf.mode == ALLINTRA) {
       x->intra_sb_rdmult_modifier = 128;

av1/encoder/nonrd_pickmode.c

diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index d08e9c0..7b204c7 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1668,6 +1668,8 @@
   if (try_palette) {
     const TxfmSearchInfo *txfm_info = &x->txfm_search_info;
     const unsigned int intra_ref_frame_cost = 0;
+    x->color_palette_thresh = (best_sad_norm < 500) ? 32 : 64;
+
     // Search palette mode for Luma plane in intra frame.
     av1_search_palette_mode_luma(cpi, x, bsize, intra_ref_frame_cost, ctx,
                                  &this_rdc, best_rdc.rdcost);

av1/encoder/palette.c

diff --git a/av1/encoder/palette.c b/av1/encoder/palette.c
index 45b5619..6ae1c6c 100644
--- a/av1/encoder/palette.c
+++ b/av1/encoder/palette.c
@@ -564,7 +564,7 @@
   }
 
   uint8_t *const color_map = xd->plane[0].color_index_map;
-  int color_thresh_palette = 64;
+  int color_thresh_palette = x->color_palette_thresh;
   // Allow for larger color_threshold for palette search, based on color,
   // scene_change, and block source variance.
   // Since palette is Y based, only allow larger threshold if block