diff --git a/av1/common/resize.c b/av1/common/resize.c
index 4188355..930eaf0 100644
--- a/av1/common/resize.c
+++ b/av1/common/resize.c
@@ -1421,12 +1421,8 @@
     }
 #endif
     return scaled;
-  } else {
-#if CONFIG_AV1_ENCODER && !CONFIG_REALTIME_ONLY
-    aom_invalidate_pyramid(unscaled->y_pyramid);
-#endif  // CONFIG_AV1_ENCODER && !CONFIG_REALTIME_ONLY
-    return unscaled;
   }
+  return unscaled;
 }
 
 // Calculates the scaled dimension given the original dimension and the scale
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 9746277..332556a 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2726,9 +2726,7 @@
       cpi->sf.interp_sf.adaptive_interp_filter_search)
     cpi->interp_search_flags.interp_filter_search_mask =
         av1_setup_interp_filter_search_mask(cpi);
-#if !CONFIG_REALTIME_ONLY
   aom_invalidate_pyramid(cpi->source->y_pyramid);
-#endif  // !CONFIG_REALTIME_ONLY
 
   av1_setup_frame_size(cpi);