diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 67f36fb..3290e26 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -1673,8 +1673,8 @@
     if (cfg->g_lag_in_frames > 1 || cfg->g_pass != AOM_RC_ONE_PASS)
       ERROR("Cannot change width or height after initialization");
     // Note: function encoder_set_config() is allowed to be called multiple
-    // times. In single-pass realtime mode without lookahead (g_lag_in_frames ==
-    // 0), and with the maximum frame size declared up front via
+    // times. In one-pass mode without lookahead (g_lag_in_frames == 0),
+    // and with the maximum frame size declared up front via
     // g_forced_max_frame_width/height, reference frame scaling allows upscaling
     // up to 16x and downscaling by up to 2x without forcing a keyframe. The
     // forced maximum frame size is required because the internal buffers are
@@ -1689,8 +1689,7 @@
     // actual coded frame size.
     const bool allow_ref_scaled_upscale =
         cfg->g_forced_max_frame_width && cfg->g_forced_max_frame_height &&
-        ctx->oxcf.mode == REALTIME && cfg->g_pass == AOM_RC_ONE_PASS &&
-        cfg->g_lag_in_frames == 0;
+        cfg->g_pass == AOM_RC_ONE_PASS && cfg->g_lag_in_frames == 0;
     if (ctx->ppi->cpi->svc.number_spatial_layers == 1 &&
         ctx->ppi->cpi->last_coded_width && ctx->ppi->cpi->last_coded_height &&
         (!valid_ref_frame_size(ctx->ppi->cpi->last_coded_width,
diff --git a/av1/encoder/motion_search_facade.c b/av1/encoder/motion_search_facade.c
index 5079ef5..8e9c428 100644
--- a/av1/encoder/motion_search_facade.c
+++ b/av1/encoder/motion_search_facade.c
@@ -692,9 +692,19 @@
     }
 
     // Do sub-pixel compound motion search on the current reference frame.
+    const struct scale_factors *orig_sf = NULL;
     if (id) {
       orig_yv12 = xd->plane[plane].pre[0];
       xd->plane[plane].pre[0] = xd->plane[plane].pre[id];
+      // Sub-pixel motion search works on raw reference frames that may not
+      // match the resolution of the current frame, so block_ref_scale_factors
+      // must be kept in sync with the frame data in order to perform the
+      // motion search correctly.
+      // Full-pixel motion search works on scaled reference frames, so it
+      // doesn't need to update block_ref_scale_factors when swapping in
+      // scaled_ref_frame.
+      orig_sf = xd->block_ref_scale_factors[0];
+      xd->block_ref_scale_factors[0] = xd->block_ref_scale_factors[id];
     }
 
     if (cpi->common.features.cur_frame_force_integer_mv) {
@@ -734,7 +744,10 @@
     }
 
     // Restore the pointer to the first prediction buffer.
-    if (id) xd->plane[plane].pre[0] = orig_yv12;
+    if (id) {
+      xd->plane[plane].pre[0] = orig_yv12;
+      xd->block_ref_scale_factors[0] = orig_sf;
+    }
     if (bestsme < last_besterr[id]) {
       cur_mv[id] = best_mv;
       last_besterr[id] = bestsme;
@@ -834,6 +847,12 @@
     }
   }
 
+  const struct scale_factors *orig_sf = NULL;
+  if (ref_idx) {
+    orig_sf = xd->block_ref_scale_factors[0];
+    xd->block_ref_scale_factors[0] = xd->block_ref_scale_factors[ref_idx];
+  }
+
   if (cpi->common.features.cur_frame_force_integer_mv) {
     convert_fullmv_to_mv(&best_mv);
   }
@@ -856,7 +875,10 @@
   }
 
   // Restore the pointer to the first unscaled prediction buffer.
-  if (ref_idx) pd->pre[0] = orig_yv12;
+  if (ref_idx) {
+    pd->pre[0] = orig_yv12;
+    xd->block_ref_scale_factors[0] = orig_sf;
+  }
 
   if (bestsme < INT_MAX) *this_mv = best_mv.as_mv;
 
diff --git a/test/frame_size_tests.cc b/test/frame_size_tests.cc
index 96d5448..97bbc59 100644
--- a/test/frame_size_tests.cc
+++ b/test/frame_size_tests.cc
@@ -223,24 +223,17 @@
       while ((pkt = aom_codec_get_cx_data(enc.get(), &iter)) != nullptr) {
         ASSERT_EQ(pkt->kind, AOM_CODEC_CX_FRAME_PKT);
         // All the resolution changes above are within the reference frame
-        // scaling limits (up to 16x up and 2x down). In single pass realtime
-        // mode without lookahead, and with the maximum frame size declared up
-        // front via g_forced_max_frame_width/height, such changes are coded as
-        // inter frames that scale their references, so only the very first
-        // frame is a keyframe. Other modes force a keyframe on every
-        // resolution change.
-        const bool scales_references = usage_ == AOM_USAGE_REALTIME;
+        // scaling limits (up to 16x up and 2x down). In one pass mode
+        // without lookahead, and with the maximum frame size declared up front
+        // via g_forced_max_frame_width/height, such changes are coded as inter
+        // frames that scale their references, so only the very first frame is
+        // a keyframe. All intra mode codes every frame as a keyframe.
         if (usage_ == AOM_USAGE_ALL_INTRA || frame_count == 0) {
           EXPECT_NE(pkt->data.frame.flags & AOM_FRAME_IS_KEY, 0u)
               << "frame " << frame_count;
-        } else if (i == 0) {
-          if (scales_references) {
-            EXPECT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_KEY, 0u)
-                << "frame " << frame_count;
-          } else {
-            EXPECT_NE(pkt->data.frame.flags & AOM_FRAME_IS_KEY, 0u)
-                << "frame " << frame_count;
-          }
+        } else {
+          EXPECT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_KEY, 0u)
+              << "frame " << frame_count;
         }
         frame_count++;
       }