diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 69076a8..a44a733 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -457,6 +457,9 @@
   if (AOMMIN(cm->width, cm->height) > 1080) {
     sf->use_upsampled_references = 0;
   }
+  if ((AOMMIN(cm->width, cm->height) > 720) && (oxcf->profile != PROFILE_0)) {
+    sf->use_upsampled_references = 0;
+  }
 
   if (oxcf->mode == REALTIME) {
     set_rt_speed_feature_framesize_dependent(cpi, sf, oxcf->speed);