diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 3f93a8e..a8cc8ff 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -472,6 +472,8 @@
   // The type of mv cost used during motion search
   MV_COST_TYPE mv_cost_type;
 
+  int search_ref_frame[REF_FRAMES];
+
 #if CONFIG_AV1_HIGHBITDEPTH
   void (*fwd_txfm4x4)(const int16_t *input, tran_low_t *output, int stride);
   void (*inv_txfm_add)(const tran_low_t *input, uint8_t *dest, int stride,
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 9436c37..687d688 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4388,6 +4388,75 @@
 #endif  // CONFIG_INTERNAL_STATS
 }
 
+#if !CONFIG_REALTIME_ONLY
+static void init_ref_frame_space(AV1_COMP *cpi, ThreadData *td, int mi_row,
+                                 int mi_col) {
+  const AV1_COMMON *cm = &cpi->common;
+  MACROBLOCK *x = &td->mb;
+  const int frame_idx = cpi->gf_group.index;
+  TplDepFrame *tpl_frame = &cpi->tpl_frame[frame_idx];
+
+  memset(x->search_ref_frame, 0, sizeof(x->search_ref_frame));
+
+  if (tpl_frame->is_valid == 0) return;
+  if (!is_frame_tpl_eligible(cpi)) return;
+  if (frame_idx >= MAX_LAG_BUFFERS) return;
+  if (cpi->oxcf.superres_mode != SUPERRES_NONE) return;
+  if (cpi->oxcf.aq_mode != NO_AQ) return;
+
+  TplDepStats *tpl_stats = tpl_frame->tpl_stats_ptr;
+  const int tpl_stride = tpl_frame->stride;
+  int64_t inter_cost[INTER_REFS_PER_FRAME] = { 0 };
+  const int step = 1 << cpi->tpl_stats_block_mis_log2;
+
+  const BLOCK_SIZE sb_size = cm->seq_params.sb_size;
+  const int mi_row_end = AOMMIN(mi_size_high[sb_size] + mi_row, cm->mi_rows);
+  const int mi_col_end = AOMMIN(mi_size_wide[sb_size] + mi_col, cm->mi_cols);
+
+  for (int row = mi_row; row < mi_row_end; row += step) {
+    for (int col = mi_col; col < mi_col_end; col += step) {
+      TplDepStats *this_stats =
+          &tpl_stats[av1_tpl_ptr_pos(cpi, row, col, tpl_stride)];
+      for (int rf_idx = 0; rf_idx < INTER_REFS_PER_FRAME; ++rf_idx)
+        inter_cost[rf_idx] += this_stats->pred_error[rf_idx];
+    }
+  }
+
+  int rank_index[INTER_REFS_PER_FRAME - 1] = { 0 };
+
+  for (int idx = 0; idx < INTER_REFS_PER_FRAME - 1; ++idx) {
+    rank_index[idx] = idx + 1;
+    for (int i = idx; i > 0; --i) {
+      if (inter_cost[rank_index[i - 1]] > inter_cost[rank_index[i]]) {
+        const int tmp = rank_index[i - 1];
+        rank_index[i - 1] = rank_index[i];
+        rank_index[i] = tmp;
+      }
+    }
+  }
+
+  const int is_overlay = cpi->gf_group.update_type[frame_idx] == OVERLAY_UPDATE;
+  x->search_ref_frame[INTRA_FRAME] = 1;
+  x->search_ref_frame[LAST_FRAME] = 1;
+
+  int cutoff_ref = 0;
+
+  for (int idx = 0; idx < INTER_REFS_PER_FRAME - 1; ++idx) {
+    x->search_ref_frame[rank_index[idx] + LAST_FRAME] = 1;
+    if (idx > 2 && !is_overlay) {
+      // If the predictive coding gains are smaller than the previous more
+      // relevant frame over certain amount, discard this frame.
+      if (labs(inter_cost[rank_index[idx]]) <
+              labs(inter_cost[rank_index[idx - 1]]) / 8 ||
+          inter_cost[rank_index[idx]] == 0)
+        cutoff_ref = 1;
+
+      if (cutoff_ref) x->search_ref_frame[rank_index[idx] + LAST_FRAME] = 0;
+    }
+  }
+}
+#endif  // !CONFIG_REALTIME_ONLY
+
 // This function initializes the stats for encode_rd_sb.
 static INLINE void init_encode_rd_sb(AV1_COMP *cpi, ThreadData *td,
                                      const TileDataEnc *tile_data,
@@ -4410,6 +4479,7 @@
   }
 
 #if !CONFIG_REALTIME_ONLY
+  init_ref_frame_space(cpi, td, mi_row, mi_col);
   x->sb_energy_level = 0;
   x->cnn_output_valid = 0;
   if (gather_tpl_data) {
@@ -5202,7 +5272,7 @@
         ref_buf[frame]->y_crop_height == cpi->source->y_crop_height &&
         do_gm_search_logic(&cpi->sf, frame) &&
         !prune_ref_by_selective_ref_frame(
-            cpi, ref_frame, cm->cur_frame->ref_display_order_hint,
+            cpi, NULL, ref_frame, cm->cur_frame->ref_display_order_hint,
             cm->current_frame.display_order_hint) &&
         !(cpi->sf.gm_sf.selective_ref_gm && skip_gm_frame(cm, frame))) {
       assert(ref_buf[frame] != NULL);
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 86fd720..7b6e685 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -163,6 +163,7 @@
   int64_t mc_dep_dist;
   int_mv mv[INTER_REFS_PER_FRAME];
   int ref_frame_index;
+  int64_t pred_error[INTER_REFS_PER_FRAME];
 #if !USE_TPL_CLASSIC_MODEL
   int64_t mc_count;
   int64_t mc_saved;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 994b9cf..4490591 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3557,7 +3557,7 @@
     }
   }
 
-  if (prune_ref_by_selective_ref_frame(cpi, ref_frame,
+  if (prune_ref_by_selective_ref_frame(cpi, x, ref_frame,
                                        cm->cur_frame->ref_display_order_hint,
                                        cm->current_frame.display_order_hint))
     return 1;
diff --git a/av1/encoder/rdopt.h b/av1/encoder/rdopt.h
index 3a59ea4..22f83ff 100644
--- a/av1/encoder/rdopt.h
+++ b/av1/encoder/rdopt.h
@@ -159,6 +159,8 @@
                             const unsigned int frame_display_order_hint,
                             const int *ref_frame_list) {
   for (int i = 0; i < 2; i++) {
+    if (ref_frame_list[i] == NONE_FRAME) continue;
+
     if (ref_frame[0] == ref_frame_list[i] ||
         ref_frame[1] == ref_frame_list[i]) {
       if (av1_encoder_get_relative_dist(
@@ -172,10 +174,12 @@
 }
 
 static INLINE int prune_ref_by_selective_ref_frame(
-    const AV1_COMP *const cpi, const MV_REFERENCE_FRAME *const ref_frame,
+    const AV1_COMP *const cpi, const MACROBLOCK *const x,
+    const MV_REFERENCE_FRAME *const ref_frame,
     const unsigned int *const ref_display_order_hint,
     const unsigned int cur_frame_display_order_hint) {
   const SPEED_FEATURES *const sf = &cpi->sf;
+
   if (sf->inter_sf.selective_ref_frame) {
     const AV1_COMMON *const cm = &cpi->common;
     const OrderHintInfo *const order_hint_info =
@@ -183,7 +187,13 @@
     const int comp_pred = ref_frame[1] > INTRA_FRAME;
     if (sf->inter_sf.selective_ref_frame >= 2 ||
         (sf->inter_sf.selective_ref_frame == 1 && comp_pred)) {
-      const int ref_frame_list[2] = { LAST3_FRAME, LAST2_FRAME };
+      int ref_frame_list[2] = { LAST3_FRAME, LAST2_FRAME };
+
+      if (x != NULL) {
+        if (x->search_ref_frame[LAST3_FRAME]) ref_frame_list[0] = NONE_FRAME;
+        if (x->search_ref_frame[LAST2_FRAME]) ref_frame_list[1] = NONE_FRAME;
+      }
+
       if (prune_ref(ref_frame, order_hint_info, ref_display_order_hint,
                     ref_display_order_hint[GOLDEN_FRAME - LAST_FRAME],
                     ref_frame_list))
@@ -210,9 +220,16 @@
     }
 
     if (sf->inter_sf.selective_ref_frame >= 3) {
-      static const int ref_frame_list[2] = { ALTREF2_FRAME, BWDREF_FRAME };
+      int ref_frame_list[2] = { ALTREF2_FRAME, BWDREF_FRAME };
+
+      if (x != NULL) {
+        if (x->search_ref_frame[ALTREF2_FRAME]) ref_frame_list[0] = NONE_FRAME;
+        if (x->search_ref_frame[BWDREF_FRAME]) ref_frame_list[1] = NONE_FRAME;
+      }
+
       if (prune_ref(ref_frame, order_hint_info, ref_display_order_hint,
-                    cur_frame_display_order_hint, ref_frame_list))
+                    ref_display_order_hint[LAST_FRAME - LAST_FRAME],
+                    ref_frame_list))
         return 1;
     }
 
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index 3c244f7..7917213 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -382,6 +382,9 @@
     if (inter_cost < best_inter_cost) {
       memcpy(best_coeff, coeff, sizeof(best_coeff));
       best_rf_idx = rf_idx;
+
+      if (rf_idx == 0) tpl_stats->pred_error[rf_idx] = inter_cost;
+
       best_inter_cost = inter_cost;
       best_mv.as_int = x->best_mv.as_int;
       if (best_inter_cost < best_intra_cost) {
@@ -392,6 +395,11 @@
     }
   }
 
+  if (best_rf_idx >= 0) {
+    tpl_stats->pred_error[best_rf_idx] =
+        best_inter_cost - tpl_stats->pred_error[0];
+  }
+
   if (best_inter_cost < INT64_MAX) {
     uint16_t eob;
     get_quantize_error(x, 0, best_coeff, qcoeff, dqcoeff, tx_size, &eob,
@@ -657,6 +665,8 @@
       tpl_ptr->srcrf_rate = srcrf_rate;
       tpl_ptr->recrf_rate = recrf_rate;
       memcpy(tpl_ptr->mv, src_stats->mv, sizeof(tpl_ptr->mv));
+      memcpy(tpl_ptr->pred_error, src_stats->pred_error,
+             sizeof(tpl_ptr->pred_error));
       tpl_ptr->ref_frame_index = src_stats->ref_frame_index;
       ++tpl_ptr;
     }
@@ -672,8 +682,6 @@
   const YV12_BUFFER_CONFIG *ref_frame[7] = { NULL, NULL, NULL, NULL,
                                              NULL, NULL, NULL };
   const YV12_BUFFER_CONFIG *ref_frames_ordered[INTER_REFS_PER_FRAME];
-  unsigned int ref_frame_display_index[7];
-  MV_REFERENCE_FRAME ref[2] = { LAST_FRAME, INTRA_FRAME };
   int ref_frame_flags;
   const YV12_BUFFER_CONFIG *src_frame[7] = { NULL, NULL, NULL, NULL,
                                              NULL, NULL, NULL };
@@ -700,9 +708,7 @@
   xd->cur_buf = this_frame;
 
   for (idx = 0; idx < INTER_REFS_PER_FRAME; ++idx) {
-    TplDepFrame *tpl_ref_frame = &cpi->tpl_frame[tpl_frame->ref_map_index[idx]];
     ref_frame[idx] = cpi->tpl_frame[tpl_frame->ref_map_index[idx]].rec_picture;
-    ref_frame_display_index[idx] = tpl_ref_frame->frame_display_index;
     src_frame[idx] = cpi->tpl_frame[tpl_frame->ref_map_index[idx]].gf_picture;
   }
 
@@ -723,16 +729,6 @@
     }
   }
 
-  // Skip motion estimation w.r.t. reference frames which are not
-  // considered in RD search, using "selective_ref_frame" speed feature
-  for (idx = 0; idx < INTER_REFS_PER_FRAME; ++idx) {
-    ref[0] = idx + 1;
-    if (prune_ref_by_selective_ref_frame(cpi, ref, ref_frame_display_index,
-                                         tpl_frame->frame_display_index)) {
-      ref_frame[idx] = NULL;
-    }
-  }
-
   // Make a temporary mbmi for tpl model
   MB_MODE_INFO mbmi;
   memset(&mbmi, 0, sizeof(mbmi));