diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index c346c92..a6c0cd6 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4795,6 +4795,16 @@
       return 1;
   }
 
+  if (sf->rt_sf.prune_inter_modes_with_golden_ref &&
+      ref_frame == GOLDEN_FRAME && !comp_pred) {
+    const int subgop_size = AOMMIN(cpi->ppi->gf_group.size, FIXED_GF_INTERVAL);
+    if (cpi->rc.frames_since_golden > (subgop_size >> 2) &&
+        args->search_state->best_mbmode.ref_frame[0] != GOLDEN_FRAME) {
+      if ((bsize > BLOCK_16X16 && this_mode == NEWMV) || this_mode == NEARMV)
+        return 1;
+    }
+  }
+
   return 0;
 }
 
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index cd4a439..9199b62 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -1405,6 +1405,7 @@
     sf->rt_sf.skip_interp_filter_search = 1;
     sf->rt_sf.use_real_time_ref_set = 1;
     sf->rt_sf.use_simple_rd_model = 1;
+    sf->rt_sf.prune_inter_modes_with_golden_ref = boosted ? 0 : 1;
 
     sf->rt_sf.check_scene_detection = 1;
     if (cm->current_frame.frame_type != KEY_FRAME &&
@@ -1847,6 +1848,7 @@
   rt_sf->skip_tx_no_split_var_based_partition = 0;
   rt_sf->skip_newmv_mode_based_on_sse = 0;
   rt_sf->gf_length_lvl = 0;
+  rt_sf->prune_inter_modes_with_golden_ref = 0;
 }
 
 void av1_set_speed_features_framesize_dependent(AV1_COMP *cpi, int speed) {
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 6202191..d8ea242 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -1317,6 +1317,9 @@
   // Define gf length multiplier.
   // Level 0: use large multiplier, level 1: use medium multiplier.
   int gf_length_lvl;
+
+  // Prune inter modes with golden frame as reference for NEARMV and NEWMV modes
+  int prune_inter_modes_with_golden_ref;
 } REAL_TIME_SPEED_FEATURES;
 
 /*!\endcond */