Re-order intra modes during RD search

Prioritize modes that are more likely to be picked. Impact on
compression is neutral. Keyframe encoding speed increases by about 5%.

Change-Id: I57f7481a42b833203e4cfb84806fe755ec37d615
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 12b04c8..47ff845 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -499,6 +499,18 @@
 #endif  // CONFIG_EXT_INTER
 };
 
+static const PREDICTION_MODE intra_rd_search_mode_order[INTRA_MODES] = {
+  DC_PRED,       H_PRED,        V_PRED,
+#if CONFIG_ALT_INTRA
+  SMOOTH_PRED,
+#endif  // CONFIG_ALT_INTRA
+  TM_PRED,
+#if CONFIG_ALT_INTRA && CONFIG_SMOOTH_HV
+  SMOOTH_V_PRED, SMOOTH_H_PRED,
+#endif  // CONFIG_ALT_INTRA && CONFIG_SMOOTH_HV
+  D135_PRED,     D207_PRED,     D153_PRED, D63_PRED, D117_PRED, D45_PRED,
+};
+
 #if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE
 static INLINE int write_uniform_cost(int n, int v) {
   const int l = get_unsigned_bits(n);
@@ -3902,7 +3914,6 @@
                                       int *rate, int *rate_tokenonly,
                                       int64_t *distortion, int *skippable,
                                       BLOCK_SIZE bsize, int64_t best_rd) {
-  uint8_t mode_idx;
   MACROBLOCKD *const xd = &x->e_mbd;
   MODE_INFO *const mic = xd->mi[0];
   MB_MODE_INFO *const mbmi = &mic->mbmi;
@@ -3977,7 +3988,7 @@
     x->use_default_intra_tx_type = 0;
 
   /* Y Search for intra prediction mode */
-  for (mode_idx = DC_PRED; mode_idx <= FINAL_MODE_SEARCH; ++mode_idx) {
+  for (int mode_idx = DC_PRED; mode_idx <= FINAL_MODE_SEARCH; ++mode_idx) {
     RD_STATS this_rd_stats;
     int this_rate, this_rate_tokenonly, s;
     int64_t this_distortion, this_rd, this_model_rd;
@@ -3986,7 +3997,8 @@
       mbmi->mode = best_mbmi.mode;
       x->use_default_intra_tx_type = 0;
     } else {
-      mbmi->mode = mode_idx;
+      assert(mode_idx < INTRA_MODES);
+      mbmi->mode = intra_rd_search_mode_order[mode_idx];
     }
 #if CONFIG_PVQ
     od_encode_rollback(&x->daala_enc, &pre_buf);
@@ -5279,20 +5291,19 @@
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   assert(!is_inter_block(mbmi));
   MB_MODE_INFO best_mbmi = *mbmi;
-  PREDICTION_MODE mode;
   int64_t best_rd = INT64_MAX, this_rd;
-  int this_rate;
-  RD_STATS tokenonly_rd_stats;
 #if CONFIG_PVQ
   od_rollback_buffer buf;
   od_encode_checkpoint(&x->daala_enc, &buf);
 #endif  // CONFIG_PVQ
 #if CONFIG_PALETTE
   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
-  uint8_t *best_palette_color_map = NULL;
 #endif  // CONFIG_PALETTE
 
-  for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
+  for (int mode_idx = DC_PRED; mode_idx < INTRA_MODES; ++mode_idx) {
+    int this_rate;
+    RD_STATS tokenonly_rd_stats;
+    PREDICTION_MODE mode = intra_rd_search_mode_order[mode_idx];
 #if CONFIG_EXT_INTRA
     const int is_directional_mode =
         av1_is_directional_mode(mode, mbmi->sb_type);
@@ -5358,7 +5369,7 @@
 
 #if CONFIG_PALETTE
   if (cpi->common.allow_screen_content_tools && mbmi->sb_type >= BLOCK_8X8) {
-    best_palette_color_map = x->palette_buffer->best_palette_color_map;
+    uint8_t *best_palette_color_map = x->palette_buffer->best_palette_color_map;
     rd_pick_palette_intra_sbuv(cpi, x,
                                cpi->intra_uv_mode_cost[mbmi->mode][DC_PRED],
                                best_palette_color_map, &best_mbmi, &best_rd,