diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 0ea2873..51c95e2 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -5155,7 +5155,8 @@
 
   cm->mi_params.setup_mi(&cm->mi_params);
 
-  av1_setup_motion_field(cm);
+  av1_calculate_ref_frame_side(cm);
+  if (cm->features.allow_ref_frame_mvs) av1_setup_motion_field(cm);
 
   av1_setup_block_planes(xd, cm->seq_params->subsampling_x,
                          cm->seq_params->subsampling_y, num_planes);