Skip setup_buffer_ref_mvs_inter for skipped frames

Some ref frames are skipped, setup_buffer_ref_mvs_inter
for them can be skipped too

For encoder, about 2.1% faster shows by encoding
20 frame of BasketballDrill_832x480_50.y4m at 800kbps
on speed 1.(265436ms -> 259866ms)

STATS_CHANGED expected
Change-Id: Ifdd13ab00661d74d5c8b30e7104f6f06f4504518
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index db8a261..b6b7829 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9811,6 +9811,10 @@
   mbmi->ref_frame[1] = second_ref_frame;
   const uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
   if (x->mbmi_ext->ref_mv_count[ref_frame_type] == UINT8_MAX) {
+    if (x->mbmi_ext->ref_mv_count[ref_frame] == UINT8_MAX ||
+        x->mbmi_ext->ref_mv_count[second_ref_frame] == UINT8_MAX) {
+      return;
+    }
     MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
     av1_find_mv_refs(cm, xd, mbmi, ref_frame_type, mbmi_ext->ref_mv_count,
                      mbmi_ext->ref_mv_stack, NULL, mbmi_ext->global_mvs, mi_row,
@@ -10069,7 +10073,13 @@
     x->pred_mv_sad[ref_frame] = INT_MAX;
     x->mbmi_ext->mode_context[ref_frame] = 0;
     x->mbmi_ext->compound_mode_context[ref_frame] = 0;
+    mbmi_ext->ref_mv_count[ref_frame] = UINT8_MAX;
     if (cpi->ref_frame_flags & ref_frame_flag_list[ref_frame]) {
+      if (mbmi->partition != PARTITION_NONE &&
+          mbmi->partition != PARTITION_SPLIT &&
+          (skip_ref_frame_mask & (1 << ref_frame))) {
+        continue;
+      }
       assert(get_ref_frame_buffer(cpi, ref_frame) != NULL);
       setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, mi_row, mi_col,
                                  yv12_mb);