diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index c11c21e..e8a5711 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1079,8 +1079,13 @@
 }
 
 static AOM_INLINE void setup_prune_ref_frame_mask(AV1_COMP *cpi) {
-  if (!cpi->sf.rt_sf.use_nonrd_pick_mode &&
-      cpi->sf.inter_sf.selective_ref_frame >= 2) {
+  if ((!cpi->oxcf.ref_frm_cfg.enable_onesided_comp ||
+       cpi->sf.inter_sf.disable_onesided_comp) &&
+      cpi->all_one_sided_refs) {
+    // Disable all compound references
+    cpi->prune_ref_frame_mask = (1 << MODE_CTX_REF_FRAMES) - (1 << REF_FRAMES);
+  } else if (!cpi->sf.rt_sf.use_nonrd_pick_mode &&
+             cpi->sf.inter_sf.selective_ref_frame >= 2) {
     AV1_COMMON *const cm = &cpi->common;
     const OrderHintInfo *const order_hint_info =
         &cm->seq_params.order_hint_info;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 7207ced..ebe8a44 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3684,6 +3684,22 @@
   }
 }
 
+static AOM_INLINE int prune_ref_frame(const AV1_COMP *cpi, const MACROBLOCK *x,
+                                      MV_REFERENCE_FRAME ref_frame) {
+  const AV1_COMMON *const cm = &cpi->common;
+  MV_REFERENCE_FRAME rf[2];
+  av1_set_ref_frame(rf, ref_frame);
+
+  if ((cpi->prune_ref_frame_mask >> ref_frame) & 1) return 1;
+
+  if (prune_ref_by_selective_ref_frame(cpi, x, rf,
+                                       cm->cur_frame->ref_display_order_hint)) {
+    return 1;
+  }
+
+  return 0;
+}
+
 // Please add/modify parameter setting in this function, making it consistent
 // and easy to read and maintain.
 static AOM_INLINE void set_params_rd_pick_inter_mode(
@@ -3737,7 +3753,7 @@
           AOMMIN(x->best_pred_mv_sad, x->pred_mv_sad[ref_frame]);
   }
   // ref_frame = ALTREF_FRAME
-  if (!cpi->sf.rt_sf.use_real_time_ref_set) {
+  if (!cpi->sf.rt_sf.use_real_time_ref_set && is_comp_ref_allowed(bsize)) {
     // No second reference on RT ref set, so no need to initialize
     for (; ref_frame < MODE_CTX_REF_FRAMES; ++ref_frame) {
       x->mbmi_ext->mode_context[ref_frame] = 0;
@@ -3754,6 +3770,10 @@
           continue;
         }
       }
+      // Ref mv list population is not required, when compound references are
+      // pruned.
+      if (prune_ref_frame(cpi, x, ref_frame)) continue;
+
       av1_find_mv_refs(cm, xd, mbmi, ref_frame, mbmi_ext->ref_mv_count,
                        xd->ref_mv_stack, xd->weight, NULL, mbmi_ext->global_mvs,
                        mbmi_ext->mode_context);
@@ -3970,7 +3990,7 @@
   }
 
   const int ref_type = av1_ref_frame_type(ref_frame);
-  if ((cpi->prune_ref_frame_mask >> ref_type) & 1) return 1;
+  if (prune_ref_frame(cpi, x, ref_type)) return 1;
 
   // This is only used in motion vector unit test.
   if (cpi->oxcf.unit_test_cfg.motion_vector_unit_test &&
@@ -3982,13 +4002,6 @@
     return 1;
   }
 
-  const int comp_pred = ref_frame[1] > INTRA_FRAME;
-  if ((!cpi->oxcf.ref_frm_cfg.enable_onesided_comp ||
-       cpi->sf.inter_sf.disable_onesided_comp) &&
-      comp_pred && cpi->all_one_sided_refs) {
-    return 1;
-  }
-
   const MB_MODE_INFO *const mbmi = x->e_mbd.mi[0];
   // If no valid mode has been found so far in PARTITION_NONE when finding a
   // valid partition is required, do not skip mode.
@@ -4034,10 +4047,6 @@
     }
   }
 
-  if (prune_ref_by_selective_ref_frame(cpi, x, ref_frame,
-                                       cm->cur_frame->ref_display_order_hint))
-    return 1;
-
   if (skip_motion_mode) return 2;
 
   return 0;