diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 8bdbe17..f485aba 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -480,6 +480,7 @@
   const RATE_CONTROL *const rc = &cpi->rc;
   const PRIMARY_RATE_CONTROL *const p_rc = &cpi->ppi->p_rc;
   const AV1_COMMON *const cm = &cpi->common;
+  const SVC *const svc = &cpi->svc;
   const RefreshFrameInfo *const refresh_frame = &cpi->refresh_frame;
   int max_delta_down;
   int max_delta_up = 20;
@@ -511,7 +512,7 @@
                          ? AOMMIN(8, AOMMAX(1, rc->q_1_frame / 16))
                          : AOMMIN(16, AOMMAX(1, rc->q_1_frame / 8));
   }
-  if (cpi->svc.number_temporal_layers > 1 && cpi->svc.temporal_layer_id == 0)
+  if (svc->number_temporal_layers > 1 && svc->temporal_layer_id == 0)
     max_delta_up = AOMMIN(max_delta_up, 14);
   // If resolution changes or avg_frame_bandwidth significantly changed,
   // then set this flag to indicate change in target bits per macroblock.
@@ -568,6 +569,18 @@
     else if (q - rc->q_1_frame > max_delta_up)
       q = rc->q_1_frame + max_delta_up;
   }
+  // Constrain the Q for enhancement temporal layer, relative to base TLO.
+  if (svc->number_temporal_layers > 1 && svc->temporal_layer_id > 0 &&
+      svc->spatial_layer_id == 0) {
+    // Get base temporal layer TL0.
+    const int layer = LAYER_IDS_TO_IDX(0, 0, svc->number_temporal_layers);
+    LAYER_CONTEXT *lc = &svc->layer_context[layer];
+    // lc->rc.avg_frame_bandwidth and lc->p_rc.last_q correspond to the
+    // last TL0 frame.
+    if (rc->avg_frame_bandwidth < lc->rc.avg_frame_bandwidth &&
+        q < lc->p_rc.last_q[INTER_FRAME] - 4)
+      q = lc->p_rc.last_q[INTER_FRAME] - 4;
+  }
   // For non-svc (single layer): if resolution has increased push q closer
   // to the active_worst to avoid excess overshoot.
   if (!cpi->ppi->use_svc && cm->prev_frame &&