diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index 25a6e2c..26cea2d 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -1142,25 +1142,25 @@
   src_var /= (double)blocks;
   rec_var /= (double)blocks;
 
-  // Only take action when the spatial complexity is low
-  if ((rec_var < threshold) || (src_var < threshold)) {
-    // Dont allow 0 to prevent / 0 below.
-    src_var += 0.000001;
-    rec_var += 0.000001;
+  // Dont allow 0 to prevent / 0 below.
+  src_var += 0.000001;
+  rec_var += 0.000001;
 
-    // Heavier weigth if the reconstruction has lower variance.
-    if (src_var >= rec_var) {
-      var_diff = (src_var - rec_var) * 2;
-      variance_rd_factor = 1.0 + (var_diff / src_var);
-    } else {
-      var_diff = (rec_var - src_var) / 2;
-      variance_rd_factor = 1.0 + (var_diff / src_var);
+  if (src_var >= rec_var) {
+    var_diff = (src_var - rec_var);
+    if ((var_diff > 0.5) && (rec_var < threshold)) {
+      variance_rd_factor = 1.0 + ((var_diff * 2) / src_var);
     }
-
-    // Limit adjustment;
-    variance_rd_factor = AOMMIN(3.0, variance_rd_factor);
+  } else {
+    var_diff = (rec_var - src_var);
+    if ((var_diff > 0.5) && (src_var < threshold)) {
+      variance_rd_factor = 1.0 + (var_diff / (2 * src_var));
+    }
   }
 
+  // Limit adjustment;
+  variance_rd_factor = AOMMIN(3.0, variance_rd_factor);
+
   return variance_rd_factor;
 }