diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index b7729e9..3006199 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -2782,7 +2782,8 @@
     force_mv_inter_layer = 0;
     if (cpi->ppi->use_svc && svc->spatial_layer_id > 0 &&
         ((ref_frame == LAST_FRAME && svc->skip_mvsearch_last) ||
-         (ref_frame == GOLDEN_FRAME && svc->skip_mvsearch_gf))) {
+         (ref_frame == GOLDEN_FRAME && svc->skip_mvsearch_gf) ||
+         (ref_frame == ALTREF_FRAME && svc->skip_mvsearch_altref))) {
       // Only test mode if NEARESTMV/NEARMV is (svc_mv_col, svc_mv_row),
       // otherwise set NEWMV to (svc_mv_col, svc_mv_row).
       // Skip newmv and filter search.
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c
index d0b7d23..3007e5f 100644
--- a/av1/encoder/svc_layercontext.c
+++ b/av1/encoder/svc_layercontext.c
@@ -165,6 +165,13 @@
   }
 }
 
+static AOM_INLINE bool check_ref_is_low_spatial_res_super_frame(
+    int ref_frame, const SVC *svc) {
+  int ref_frame_idx = svc->ref_idx[ref_frame - 1];
+  return svc->buffer_time_index[ref_frame_idx] == svc->current_superframe &&
+         svc->buffer_spatial_layer[ref_frame_idx] <= svc->spatial_layer_id - 1;
+}
+
 void av1_restore_layer_context(AV1_COMP *const cpi) {
   SVC *const svc = &cpi->svc;
   const AV1_COMMON *const cm = &cpi->common;
@@ -200,14 +207,15 @@
   // refreshed (i.e., buffer slot holding that reference was refreshed) on the
   // previous spatial layer(s) at the same time (current_superframe).
   if (svc->set_ref_frame_config && svc->force_zero_mode_spatial_ref) {
-    int ref_frame_idx = svc->ref_idx[LAST_FRAME - 1];
-    if (svc->buffer_time_index[ref_frame_idx] == svc->current_superframe &&
-        svc->buffer_spatial_layer[ref_frame_idx] <= svc->spatial_layer_id - 1)
+    if (check_ref_is_low_spatial_res_super_frame(LAST_FRAME, svc)) {
       svc->skip_mvsearch_last = 1;
-    ref_frame_idx = svc->ref_idx[GOLDEN_FRAME - 1];
-    if (svc->buffer_time_index[ref_frame_idx] == svc->current_superframe &&
-        svc->buffer_spatial_layer[ref_frame_idx] <= svc->spatial_layer_id - 1)
+    }
+    if (check_ref_is_low_spatial_res_super_frame(GOLDEN_FRAME, svc)) {
       svc->skip_mvsearch_gf = 1;
+    }
+    if (check_ref_is_low_spatial_res_super_frame(ALTREF_FRAME, svc)) {
+      svc->skip_mvsearch_altref = 1;
+    }
   }
 }
 
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h
index dc6906d..e3c2653 100644
--- a/av1/encoder/svc_layercontext.h
+++ b/av1/encoder/svc_layercontext.h
@@ -112,6 +112,7 @@
   unsigned char buffer_spatial_layer[REF_FRAMES];
   int skip_mvsearch_last;
   int skip_mvsearch_gf;
+  int skip_mvsearch_altref;
   int spatial_layer_fb[REF_FRAMES];
   int temporal_layer_fb[REF_FRAMES];
   int num_encoded_top_layer;