diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 74a409f..c6a31a1 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -869,10 +869,13 @@
         cpi->sf.rt_sf.nonrd_prune_ref_frame_search;
   }
 
-  set_ref_ptrs(cm, xd, mi->ref_frame[0], mi->ref_frame[1]);
-  av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL,
-                                cm->seq_params->sb_size, AOM_PLANE_Y,
-                                AOM_PLANE_Y);
+  // Only calculate the predictor for non-zero MV.
+  if (mi->mv[0].as_int != 0) {
+    set_ref_ptrs(cm, xd, mi->ref_frame[0], mi->ref_frame[1]);
+    av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL,
+                                  cm->seq_params->sb_size, AOM_PLANE_Y,
+                                  AOM_PLANE_Y);
+  }
 }
 
 int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
@@ -974,8 +977,16 @@
   if (!is_key_frame) {
     setup_planes(cpi, x, &y_sad, &y_sad_g, &ref_frame_partition, mi_row,
                  mi_col);
-    d = xd->plane[0].dst.buf;
-    dp = xd->plane[0].dst.stride;
+
+    MB_MODE_INFO *mi = xd->mi[0];
+    // Use reference SB directly for zero mv.
+    if (mi->mv[0].as_int != 0) {
+      d = xd->plane[0].dst.buf;
+      dp = xd->plane[0].dst.stride;
+    } else {
+      d = xd->plane[0].pre[0].buf;
+      dp = xd->plane[0].pre[0].stride;
+    }
   } else {
     d = AV1_VAR_OFFS;
     dp = 0;