diff --git a/av1/qmode_rc/reference_manager.cc b/av1/qmode_rc/reference_manager.cc
index eefd188..8ea96b2 100644
--- a/av1/qmode_rc/reference_manager.cc
+++ b/av1/qmode_rc/reference_manager.cc
@@ -227,10 +227,10 @@
   if (!last_queue_.empty()) {
     int this_ref_idx = last_queue_.back();
     GopFrame &this_gf_frame = ref_frame_table_[this_ref_idx];
-    // If the last (stack top) of last_queue_ is an arf frame, then move it to
-    // the backward_queue_ so it will become the golden frame of future leaf
-    // frames.
-    if (this_gf_frame.is_arf_frame) {
+    // If the last (stack top) of last_queue_ is an arf frame or first layer
+    // intermediate arf, then move it to the backward_queue_ so it will
+    // become the golden frame of future leaf frames.
+    if (this_gf_frame.is_golden_frame) {
       last_queue_.pop_back();
       backward_queue_.push_back(this_ref_idx);
     }