diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 515331d..00e7ec6 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4407,17 +4407,30 @@
 // Set the relative distance of a reference frame w.r.t. current frame
 static void set_rel_frame_dist(AV1_COMP *cpi) {
   const AV1_COMMON *const cm = &cpi->common;
-  const SPEED_FEATURES *const sf = &cpi->sf;
   const OrderHintInfo *const order_hint_info = &cm->seq_params.order_hint_info;
   MV_REFERENCE_FRAME ref_frame;
+  int dist;
+  int min_past_dist = INT32_MAX, min_future_dist = INT32_MAX;
+  cpi->nearest_past_ref = NONE_FRAME;
+  cpi->nearest_future_ref = NONE_FRAME;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     cpi->ref_relative_dist[ref_frame - LAST_FRAME] = 0;
-    if (sf->alt_ref_search_fp) {
-      int dist = av1_encoder_get_relative_dist(
+    if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ref_frame]) {
+      dist = av1_encoder_get_relative_dist(
           order_hint_info,
           cm->cur_frame->ref_display_order_hint[ref_frame - LAST_FRAME],
           cm->current_frame.display_order_hint);
       cpi->ref_relative_dist[ref_frame - LAST_FRAME] = dist;
+      // Get the nearest ref_frame in the past
+      if (abs(dist) < min_past_dist && dist < 0) {
+        cpi->nearest_past_ref = ref_frame;
+        min_past_dist = abs(dist);
+      }
+      // Get the nearest ref_frame in the future
+      if (dist < min_future_dist && dist > 0) {
+        cpi->nearest_future_ref = ref_frame;
+        min_future_dist = dist;
+      }
     }
   }
 }
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 6426bb9..1800e78 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -1043,6 +1043,10 @@
   // Indicates the true relative distance of ref frame w.r.t. current frame
   int8_t ref_relative_dist[INTER_REFS_PER_FRAME];
 
+  // Indicate nearest references w.r.t. current frame in past and future
+  int8_t nearest_past_ref;
+  int8_t nearest_future_ref;
+
   double *ssim_rdmult_scaling_factors;
 
   // Whether writing to bitstream. It allows us to encode one frame multiple
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 25db10c..4cc4f82 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -10174,14 +10174,15 @@
 
 // Whether this reference motion vector can be skipped, based on initial
 // heuristics.
-static bool ref_mv_idx_early_breakout(MACROBLOCK *x,
-                                      const SPEED_FEATURES *const sf,
+static bool ref_mv_idx_early_breakout(const AV1_COMP *const cpi, MACROBLOCK *x,
                                       const HandleInterModeArgs *const args,
                                       int64_t ref_best_rd, int ref_mv_idx) {
+  const SPEED_FEATURES *const sf = &cpi->sf;
   MACROBLOCKD *xd = &x->e_mbd;
   MB_MODE_INFO *mbmi = xd->mi[0];
   const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
   const int8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
+  const int is_comp_pred = has_second_ref(mbmi);
   if (sf->reduce_inter_modes && ref_mv_idx > 0) {
     if (mbmi->ref_frame[0] == LAST2_FRAME ||
         mbmi->ref_frame[0] == LAST3_FRAME ||
@@ -10193,8 +10194,19 @@
         return true;
       }
     }
+    // TODO(any): Experiment with reduce_inter_modes for compound prediction
+    if (sf->reduce_inter_modes >= 2 && !is_comp_pred &&
+        have_newmv_in_inter_mode(mbmi->mode)) {
+      if (mbmi->ref_frame[0] != cpi->nearest_past_ref &&
+          mbmi->ref_frame[0] != cpi->nearest_future_ref) {
+        const int has_nearmv = have_nearmv_in_inter_mode(mbmi->mode) ? 1 : 0;
+        if (mbmi_ext->weight[ref_frame_type][ref_mv_idx + has_nearmv] <
+            REF_CAT_LEVEL) {
+          return true;
+        }
+      }
+    }
   }
-  const int is_comp_pred = has_second_ref(mbmi);
   if (sf->prune_single_motion_modes_by_simple_trans && !is_comp_pred &&
       args->single_ref_first_pass == 0) {
     if (args->simple_rd_state[ref_mv_idx].early_skipped) {
@@ -10323,7 +10335,7 @@
   // Only search indices if they have some chance of being good.
   int good_indices = 0;
   for (int i = 0; i < ref_set; ++i) {
-    if (ref_mv_idx_early_breakout(x, &cpi->sf, args, ref_best_rd, i)) {
+    if (ref_mv_idx_early_breakout(cpi, x, args, ref_best_rd, i)) {
       continue;
     }
     mask_set_bit(&good_indices, i);
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 8ba5886..4fc4271 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -318,6 +318,7 @@
   }
 
   if (speed >= 3) {
+    sf->reduce_inter_modes = boosted ? 1 : 2;
     sf->tx_size_search_method = boosted ? USE_FULL_RD : USE_LARGESTALL;
     sf->less_rectangular_check_level = 2;
     // adaptive_motion_search breaks encoder multi-thread tests.