diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 067ed15..965e4cc 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -256,6 +256,7 @@
   unsigned int simple_motion_pred_sse;
   unsigned int pred_sse[REF_FRAMES];
   int pred_mv_sad[REF_FRAMES];
+  int best_pred_mv_sad;
 
   int nmv_vec_cost[MV_JOINTS];
   int *nmvcost[2];
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 0a662f7..8ce5a11 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4423,6 +4423,24 @@
                 cpi->oxcf.max_reference_frames);
 }
 
+// Set the relative distance of a reference frame w.r.t. current frame
+static void set_rel_frame_dist(AV1_COMP *cpi) {
+  const AV1_COMMON *const cm = &cpi->common;
+  const SPEED_FEATURES *const sf = &cpi->sf;
+  const OrderHintInfo *const order_hint_info = &cm->seq_params.order_hint_info;
+  MV_REFERENCE_FRAME ref_frame;
+  for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
+    cpi->ref_relative_dist[ref_frame - LAST_FRAME] = 0;
+    if (sf->alt_ref_search_fp) {
+      int dist = av1_encoder_get_relative_dist(
+          order_hint_info,
+          cm->cur_frame->ref_display_order_hint[ref_frame - LAST_FRAME],
+          cm->current_frame.display_order_hint);
+      cpi->ref_relative_dist[ref_frame - LAST_FRAME] = dist;
+    }
+  }
+}
+
 // Enforce the number of references for each arbitrary frame based on user
 // options and speed.
 static void enforce_max_ref_frames(AV1_COMP *cpi) {
@@ -5004,6 +5022,7 @@
 
   av1_setup_frame_buf_refs(cm);
   enforce_max_ref_frames(cpi);
+  set_rel_frame_dist(cpi);
   av1_setup_frame_sign_bias(cm);
 
 #if CHECK_PRECOMPUTED_REF_FRAME_MAP
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index a3bb24b..17dbcc8 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -1037,6 +1037,9 @@
   // whether any no-zero delta_q was actually used
   int deltaq_used;
 
+  // Indicates the true relative distance of ref frame w.r.t. current frame
+  int8_t ref_relative_dist[INTER_REFS_PER_FRAME];
+
   double *ssim_rdmult_scaling_factors;
 
   // Whether writing to bitstream. It allows us to encode one frame multiple
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index b3804b5..83c210d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -11479,10 +11479,18 @@
     }
   }
 
-  if (sf->alt_ref_search_fp)
-    if (!cm->show_frame && x->pred_mv_sad[GOLDEN_FRAME] < INT_MAX)
-      if (x->pred_mv_sad[ALTREF_FRAME] > (x->pred_mv_sad[GOLDEN_FRAME] << 1))
-        mask->pred_modes[ALTREF_FRAME] |= INTER_ALL;
+  if (sf->alt_ref_search_fp) {
+    if (!cm->show_frame && x->best_pred_mv_sad < INT_MAX) {
+      int sad_thresh = x->best_pred_mv_sad + (x->best_pred_mv_sad >> 3);
+      // Conservatively skip the modes w.r.t. BWDREF, ALTREF2 and ALTREF, if
+      // those are past frames
+      for (ref_frame = BWDREF_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
+        if (cpi->ref_relative_dist[ref_frame - LAST_FRAME] < 0)
+          if (x->pred_mv_sad[ref_frame] > sad_thresh)
+            mask->pred_modes[ref_frame] |= INTER_ALL;
+      }
+    }
+  }
 
   if (sf->adaptive_mode_search) {
     if (cm->show_frame && !cpi->rc.is_src_frame_alt_ref &&
@@ -11544,6 +11552,7 @@
                            ref_costs_comp);
 
   MV_REFERENCE_FRAME ref_frame;
+  x->best_pred_mv_sad = INT_MAX;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     x->pred_mv_sad[ref_frame] = INT_MAX;
     x->mbmi_ext->mode_context[ref_frame] = 0;
@@ -11569,6 +11578,11 @@
       setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, mi_row, mi_col,
                                  yv12_mb);
     }
+    // Store the best pred_mv_sad across all past frames
+    if (cpi->sf.alt_ref_search_fp &&
+        cpi->ref_relative_dist[ref_frame - LAST_FRAME] < 0)
+      x->best_pred_mv_sad =
+          AOMMIN(x->best_pred_mv_sad, x->pred_mv_sad[ref_frame]);
   }
   // ref_frame = ALTREF_FRAME
   for (; ref_frame < MODE_CTX_REF_FRAMES; ++ref_frame) {