Properly set rate_nocoef when pallete mode is used

Change-Id: Iff04c82b3d3b5cf2c7700717c3c3d678bbbb9f9b

vp10/encoder/encodeframe.c

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 2c47be9..6379713 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -1234,7 +1234,10 @@
                                  PICK_MODE_CONTEXT *ctx,
                                  int mi_row, int mi_col, BLOCK_SIZE bsize,
                                  int output_enabled) {
-  int i, y, x_idx;
+  int y, x_idx;
+#if CONFIG_VAR_TX
+  int i;
+#endif
   VP10_COMMON *const cm = &cpi->common;
   RD_COUNTS *const rdc = &td->rd_counts;
   MACROBLOCK *const x = &td->mb;

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 3d92591..c15db0b 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -8435,6 +8435,9 @@
   if (cm->allow_screen_content_tools && !is_inter_mode(best_mbmode.mode)) {
     PREDICTION_MODE mode_selected;
     int rate2 = 0, rate_y = 0;
+#if CONFIG_SUPERTX
+    int best_rate_nocoef;
+#endif
     int64_t distortion2 = 0, distortion_y = 0, dummy_rd = best_rd, this_rd;
     int skippable = 0, rate_overhead = 0;
     TX_SIZE best_tx_size, uv_tx;
@@ -8504,8 +8507,14 @@
 
     if (skippable) {
       rate2 -= (rate_y + rate_uv_tokenonly[uv_tx]);
+#if CONFIG_SUPERTX
+      best_rate_nocoef = rate2;
+#endif
       rate2 += vp10_cost_bit(vp10_get_skip_prob(cm, xd), 1);
     } else {
+#if CONFIG_SUPERTX
+      best_rate_nocoef = rate2 - (rate_y + rate_uv_tokenonly[uv_tx]);
+#endif
       rate2 += vp10_cost_bit(vp10_get_skip_prob(cm, xd), 0);
     }
     this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
@@ -8515,6 +8524,9 @@
       mbmi->mv[0].as_int = 0;
       max_plane = 1;
       rd_cost->rate = rate2;
+#if CONFIG_SUPERTX
+      *returnrate_nocoef = best_rate_nocoef;
+#endif
       rd_cost->dist = distortion2;
       rd_cost->rdcost = this_rd;
       best_rd = this_rd;