diff --git a/av1/ratectrl_qmode.cc b/av1/ratectrl_qmode.cc
index 5f4a733..c85eebc 100644
--- a/av1/ratectrl_qmode.cc
+++ b/av1/ratectrl_qmode.cc
@@ -31,7 +31,7 @@
 
 GopFrame gop_frame_basic(int coding_idx, int order_idx, bool is_key_frame,
                          bool is_arf_frame, bool is_golden_frame,
-                         bool is_show_frame) {
+                         bool is_show_frame, int depth) {
   GopFrame gop_frame;
   gop_frame.is_valid = true;
   gop_frame.coding_idx = coding_idx;
@@ -43,6 +43,7 @@
   gop_frame.encode_ref_mode = EncodeRefMode::kRegular;
   gop_frame.colocated_ref_idx = -1;
   gop_frame.update_ref_idx = -1;
+  gop_frame.layer_depth = depth + kLayerDepthOffset;
   return gop_frame;
 }
 
@@ -58,18 +59,18 @@
   int coding_idx = static_cast<int>(gop_struct->gop_frame_list.size());
   GopFrame gop_frame;
   int num_frames = order_end - order_start;
-  // If there are less than 3 frames, stop introducing ARF
-  if (depth < max_depth && num_frames < 3) {
+  // If there are less than kMinIntervalToAddArf frames, stop introducing ARF
+  if (depth < max_depth && num_frames >= kMinIntervalToAddArf) {
     int order_mid = (order_start + order_end) / 2;
     // intermediate ARF
-    gop_frame = gop_frame_basic(coding_idx, order_mid, 0, 1, 0, 0);
+    gop_frame = gop_frame_basic(coding_idx, order_mid, 0, 1, 0, 0, depth);
     ref_frame_manager->UpdateFrame(&gop_frame, RefUpdateType::kForward,
                                    EncodeRefMode::kRegular);
     gop_struct->gop_frame_list.push_back(gop_frame);
     construct_gop_multi_layer(gop_struct, ref_frame_manager, max_depth,
                               depth + 1, order_start, order_mid);
     // show existing intermediate ARF
-    gop_frame = gop_frame_basic(coding_idx, order_mid, 0, 0, 0, 1);
+    gop_frame = gop_frame_basic(coding_idx, order_mid, 0, 0, 0, 1, max_depth);
     ref_frame_manager->UpdateFrame(&gop_frame, RefUpdateType::kNone,
                                    EncodeRefMode::kShowExisting);
     gop_struct->gop_frame_list.push_back(gop_frame);
@@ -79,7 +80,7 @@
     // regular frame
     for (int i = order_start; i < order_end; ++i) {
       coding_idx = static_cast<int>(gop_struct->gop_frame_list.size());
-      gop_frame = gop_frame_basic(coding_idx, i, 0, 0, 0, 1);
+      gop_frame = gop_frame_basic(coding_idx, i, 0, 0, 0, 1, max_depth);
       ref_frame_manager->UpdateFrame(&gop_frame, RefUpdateType::kLast,
                                      EncodeRefMode::kRegular);
       gop_struct->gop_frame_list.push_back(gop_frame);
@@ -96,26 +97,30 @@
   int coding_idx;
   GopFrame gop_frame;
   if (has_key_frame) {
+    const int key_frame_depth = -1;
     ref_frame_manager->Reset();
     coding_idx = static_cast<int>(gop_struct.gop_frame_list.size());
-    gop_frame = gop_frame_basic(coding_idx, order_start, 1, 0, 1, 1);
+    gop_frame =
+        gop_frame_basic(coding_idx, order_start, 1, 0, 1, 1, key_frame_depth);
     ref_frame_manager->UpdateFrame(&gop_frame, RefUpdateType::kBackward,
                                    EncodeRefMode::kRegular);
     gop_struct.gop_frame_list.push_back(gop_frame);
     order_start++;
   }
   // ARF
+  const int arf_depth = 0;
   coding_idx = static_cast<int>(gop_struct.gop_frame_list.size());
-  gop_frame = gop_frame_basic(coding_idx, order_arf, 0, 1, 1, 0);
+  gop_frame = gop_frame_basic(coding_idx, order_arf, 0, 1, 1, 0, arf_depth);
   ref_frame_manager->UpdateFrame(&gop_frame, RefUpdateType::kForward,
                                  EncodeRefMode::kRegular);
   gop_struct.gop_frame_list.push_back(gop_frame);
   construct_gop_multi_layer(&gop_struct, ref_frame_manager,
-                            ref_frame_manager->ForwardMaxSize(), 1, order_start,
-                            order_arf);
+                            ref_frame_manager->ForwardMaxSize(), arf_depth + 1,
+                            order_start, order_arf);
   // Overlay
   coding_idx = static_cast<int>(gop_struct.gop_frame_list.size());
-  gop_frame = gop_frame_basic(coding_idx, order_arf, 0, 0, 0, 1);
+  gop_frame = gop_frame_basic(coding_idx, order_arf, 0, 0, 0, 1,
+                              ref_frame_manager->ForwardMaxSize());
   ref_frame_manager->UpdateFrame(&gop_frame, RefUpdateType::kNone,
                                  EncodeRefMode::kOverlay);
   gop_struct.gop_frame_list.push_back(gop_frame);
diff --git a/av1/ratectrl_qmode.h b/av1/ratectrl_qmode.h
index 30ac734..6c928bf 100644
--- a/av1/ratectrl_qmode.h
+++ b/av1/ratectrl_qmode.h
@@ -21,6 +21,9 @@
 
 namespace aom {
 
+constexpr int kLayerDepthOffset = 1;
+constexpr int kMinIntervalToAddArf = 3;
+
 struct TplFrameDepStats {
   int unit_size;  // equivalent to min_block_size
   std::vector<std::vector<double>> unit_stats;
@@ -34,7 +37,7 @@
 
 GopFrame gop_frame_basic(int coding_idx, int order_idx, bool is_key_frame,
                          bool is_arf_frame, bool is_golden_frame,
-                         bool is_show_frame);
+                         bool is_show_frame, int depth);
 
 GopStruct construct_gop(RefFrameManager *ref_frame_manager,
                         int show_frame_count, bool has_key_frame);
diff --git a/test/ratectrl_qmode_test.cc b/test/ratectrl_qmode_test.cc
index 2def531..2780141 100644
--- a/test/ratectrl_qmode_test.cc
+++ b/test/ratectrl_qmode_test.cc
@@ -52,6 +52,24 @@
   }
 }
 
+void test_layer_depth(const GopStruct &gop_struct, int max_layer_depth) {
+  int gop_size = static_cast<int>(gop_struct.gop_frame_list.size());
+  for (int gop_idx = 0; gop_idx < gop_size; ++gop_idx) {
+    const auto &gop_frame = gop_struct.gop_frame_list[gop_idx];
+    if (gop_frame.is_key_frame) {
+      EXPECT_EQ(gop_frame.layer_depth, 0);
+    }
+
+    if (gop_frame.is_arf_frame) {
+      EXPECT_LT(gop_frame.layer_depth, max_layer_depth);
+    }
+
+    if (!gop_frame.is_key_frame && !gop_frame.is_arf_frame) {
+      EXPECT_EQ(gop_frame.layer_depth, max_layer_depth);
+    }
+  }
+}
+
 TEST(RateControlQModeTest, ConstructGopARF) {
   int show_frame_count = 16;
   const int max_ref_frames = 7;
@@ -61,6 +79,9 @@
       construct_gop(&ref_frame_manager, show_frame_count, has_key_frame);
   test_gop_display_order(gop_struct);
   test_colocated_show_frame(gop_struct);
+  const int max_layer_depth =
+      ref_frame_manager.ForwardMaxSize() + kLayerDepthOffset;
+  test_layer_depth(gop_struct, max_layer_depth);
 }
 
 TEST(RateControlQModeTest, ConstructGopKey) {
@@ -72,6 +93,9 @@
       construct_gop(&ref_frame_manager, show_frame_count, has_key_frame);
   test_gop_display_order(gop_struct);
   test_colocated_show_frame(gop_struct);
+  const int max_layer_depth =
+      ref_frame_manager.ForwardMaxSize() + kLayerDepthOffset;
+  test_layer_depth(gop_struct, max_layer_depth);
 }
 
 static TplBlockStats create_toy_tpl_block_stats(int h, int w, int r, int c,
@@ -136,7 +160,7 @@
   const int ref_frame_table_size = static_cast<int>(ref_frame_table.size());
   EXPECT_LE(frame_count, ref_frame_table_size);
   for (int i = 0; i < frame_count; ++i) {
-    ref_frame_table[i] = gop_frame_basic(i, i, 0, 0, 0, 1);
+    ref_frame_table[i] = gop_frame_basic(i, i, 0, 0, 0, 1, 0);
   }
   for (int i = frame_count; i < ref_frame_table_size; ++i) {
     ref_frame_table[i] = gop_frame_invalid();