diff --git a/av1/encoder/encoder_utils.c b/av1/encoder/encoder_utils.c
index 66cd272..fcc1087 100644
--- a/av1/encoder/encoder_utils.c
+++ b/av1/encoder/encoder_utils.c
@@ -571,7 +571,7 @@
                gf_group->max_layer_depth) {
       int this_height = gf_group->layer_depth[cpi->gf_frame_index];
       int arf_q = cpi->ppi->p_rc.arf_q;
-      while (this_height > 1) {
+      while (this_height > gf_group->min_layer_depth) {
         arf_q = (arf_q + cpi->oxcf.rc_cfg.cq_level + 1) / 2;
         --this_height;
       }
diff --git a/av1/encoder/firstpass.h b/av1/encoder/firstpass.h
index 078e0e5..1c620c4 100644
--- a/av1/encoder/firstpass.h
+++ b/av1/encoder/firstpass.h
@@ -352,6 +352,7 @@
   int layer_depth[MAX_STATIC_GF_GROUP_LENGTH];
   int arf_boost[MAX_STATIC_GF_GROUP_LENGTH];
   int max_layer_depth;
+  int min_layer_depth;
   int max_layer_depth_allowed;
   // This is currently only populated for AOM_Q mode
   int q_val[MAX_STATIC_GF_GROUP_LENGTH];
diff --git a/av1/encoder/gop_structure.c b/av1/encoder/gop_structure.c
index 46d4845..64c86d6 100644
--- a/av1/encoder/gop_structure.c
+++ b/av1/encoder/gop_structure.c
@@ -12,6 +12,7 @@
 #include <stdint.h>
 
 #include "av1/common/blockd.h"
+#include "av1/encoder/ratectrl.h"
 #include "config/aom_config.h"
 #include "config/aom_scale_rtcd.h"
 
@@ -582,6 +583,8 @@
 #endif  // CONFIG_FRAME_PARALLEL_ENCODE
 
   const int kf_decomp = cpi->oxcf.kf_cfg.enable_keyframe_filtering > 1;
+
+  gf_group->min_layer_depth = MAX_ARF_LAYERS;
   if (first_frame_update_type == KF_UPDATE) {
     gf_group->update_type[frame_index] = kf_decomp ? ARF_UPDATE : KF_UPDATE;
     gf_group->arf_src_offset[frame_index] = 0;
@@ -590,6 +593,7 @@
     gf_group->frame_type[frame_index] = KEY_FRAME;
     gf_group->refbuf_state[frame_index] = REFBUF_RESET;
     gf_group->max_layer_depth = 0;
+    gf_group->min_layer_depth = 0;
 #if CONFIG_FRAME_PARALLEL_ENCODE
 #if CONFIG_FRAME_PARALLEL_ENCODE_2
     gf_group->display_idx[frame_index] = cur_disp_index;
@@ -606,6 +610,7 @@
       gf_group->frame_type[frame_index] = INTER_FRAME;
       gf_group->refbuf_state[frame_index] = REFBUF_UPDATE;
       gf_group->max_layer_depth = 0;
+      gf_group->min_layer_depth = 0;
 #if CONFIG_FRAME_PARALLEL_ENCODE
 #if CONFIG_FRAME_PARALLEL_ENCODE_2
       gf_group->display_idx[frame_index] = cur_disp_index;
@@ -625,6 +630,7 @@
     gf_group->frame_type[frame_index] = INTER_FRAME;
     gf_group->refbuf_state[frame_index] = REFBUF_UPDATE;
     gf_group->max_layer_depth = 0;
+    gf_group->min_layer_depth = 0;
 #if CONFIG_FRAME_PARALLEL_ENCODE
 #if CONFIG_FRAME_PARALLEL_ENCODE_2
     gf_group->display_idx[frame_index] = cur_disp_index;
@@ -648,6 +654,7 @@
     gf_group->frame_type[frame_index] = is_fwd_kf ? KEY_FRAME : INTER_FRAME;
     gf_group->refbuf_state[frame_index] = REFBUF_UPDATE;
     gf_group->max_layer_depth = 1;
+    gf_group->min_layer_depth = AOMMIN(gf_group->min_layer_depth, 1);
     gf_group->arf_index = frame_index;
 #if CONFIG_FRAME_PARALLEL_ENCODE
 #if CONFIG_FRAME_PARALLEL_ENCODE_2
@@ -787,6 +794,7 @@
       gf_group->frame_type[frame_index] = INTER_FRAME;
       gf_group->refbuf_state[frame_index] = REFBUF_UPDATE;
       gf_group->max_layer_depth = AOMMAX(gf_group->max_layer_depth, 2);
+      gf_group->min_layer_depth = AOMMIN(gf_group->min_layer_depth, 2);
 #if CONFIG_FRAME_PARALLEL_ENCODE
       set_src_offset(gf_group, &first_frame_index, cur_frame_index,
                      frame_index);
@@ -839,6 +847,7 @@
     gf_group->frame_type[gf_idx] = INTER_FRAME;
     gf_group->refbuf_state[gf_idx] = REFBUF_UPDATE;
     gf_group->max_layer_depth = AOMMAX(gf_group->max_layer_depth, 2);
+    gf_group->min_layer_depth = AOMMIN(gf_group->min_layer_depth, 2);
   }
 
   return frame_index;