diff --git a/av1/ratectrl_qmode.cc b/av1/ratectrl_qmode.cc
index 9e2aae2..db92483 100644
--- a/av1/ratectrl_qmode.cc
+++ b/av1/ratectrl_qmode.cc
@@ -854,7 +854,8 @@
     const FirstpassInfo &firstpass_info) {
   const int stats_size = static_cast<int>(firstpass_info.stats_list.size());
   GopStructList gop_list;
-  RefFrameManager ref_frame_manager(rc_param_.ref_frame_table_size);
+  RefFrameManager ref_frame_manager(rc_param_.ref_frame_table_size,
+                                    rc_param_.max_ref_frames);
 
   int global_coding_idx_offset = 0;
   int global_order_idx_offset = 0;
diff --git a/av1/reference_manager.cc b/av1/reference_manager.cc
index 7ebaf0f..2a7fd7b 100644
--- a/av1/reference_manager.cc
+++ b/av1/reference_manager.cc
@@ -179,7 +179,8 @@
   int ref_frame_count = 0;
   int round_robin_idx = 0;
   std::set<ReferenceName> used_name_set;
-  while (ref_frame_count < available_ref_frames) {
+  while (ref_frame_count < available_ref_frames &&
+         ref_frame_count < max_ref_frames_) {
     const RefUpdateType ref_update_type = round_robin_list[round_robin_idx];
     int priority_idx = priority_idx_list[round_robin_idx];
     int ref_idx = GetRefFrameIdxByPriority(ref_update_type, priority_idx);
@@ -270,13 +271,14 @@
     const GopFrame &gop_frame) const {
   assert(gop_frame.is_valid);
   std::vector<std::pair<PrimaryRefKey, int>> candidate_list;
-  for (int ref_idx = 0; ref_idx < static_cast<int>(ref_frame_table_.size());
-       ++ref_idx) {
-    const GopFrame &ref_frame = ref_frame_table_[ref_idx];
+  for (auto &ref_frame_in_gop_frame : gop_frame.ref_frame_list) {
+    const GopFrame &ref_frame = ref_frame_table_[ref_frame_in_gop_frame.index];
     if (ref_frame.is_valid) {
-      assert(ref_idx == ref_frame.update_ref_idx);
+      assert(ref_frame_in_gop_frame.index == ref_frame.update_ref_idx);
       PrimaryRefKey key = get_primary_ref_key(gop_frame, ref_frame);
-      std::pair<PrimaryRefKey, int> candidate = { key, ref_idx };
+      std::pair<PrimaryRefKey, int> candidate = {
+        key, ref_frame_in_gop_frame.index
+      };
       candidate_list.push_back(candidate);
     }
   }
@@ -284,11 +286,10 @@
   std::sort(candidate_list.begin(), candidate_list.end());
 
   ReferenceFrame ref_frame = { -1, ReferenceName::kNoneFrame };
-  std::vector<ReferenceFrame> ref_frame_list = GetRefFrameListByPriority();
-  assert(candidate_list.size() == ref_frame_list.size());
+  assert(candidate_list.size() == gop_frame.ref_frame_list.size());
   if (!candidate_list.empty()) {
     int ref_idx = candidate_list[0].second;
-    for (const auto &frame : ref_frame_list) {
+    for (const auto &frame : gop_frame.ref_frame_list) {
       if (frame.index == ref_idx) {
         ref_frame = frame;
       }
diff --git a/av1/reference_manager.h b/av1/reference_manager.h
index 59bfda3..2ddae3a 100644
--- a/av1/reference_manager.h
+++ b/av1/reference_manager.h
@@ -24,8 +24,9 @@
 
 class RefFrameManager {
  public:
-  explicit RefFrameManager(int ref_frame_table_size)
-      : ref_frame_table_(ref_frame_table_size) {
+  explicit RefFrameManager(int ref_frame_table_size, int max_ref_frames)
+      : ref_frame_table_(ref_frame_table_size),
+        max_ref_frames_(max_ref_frames) {
     // forward_max_size_ define max number of arf frames that can exists at
     // the same time. In the other words, it's the max size of forward_stack_.
     // TODO(angiebird): Figure out if this number is optimal.
@@ -80,6 +81,7 @@
   int forward_max_size_;
   int cur_global_order_idx_;
   RefFrameTable ref_frame_table_;
+  int max_ref_frames_;
   std::deque<int> free_ref_idx_list_;
   std::vector<int> forward_stack_;
   std::deque<int> backward_queue_;
diff --git a/test/ratectrl_qmode_test.cc b/test/ratectrl_qmode_test.cc
index dd29ae9..6fd4c64 100644
--- a/test/ratectrl_qmode_test.cc
+++ b/test/ratectrl_qmode_test.cc
@@ -246,7 +246,7 @@
   const bool has_key_frame = false;
   const int global_coding_idx_offset = 5;
   const int global_order_idx_offset = 20;
-  RefFrameManager ref_frame_manager(kRefFrameTableSize);
+  RefFrameManager ref_frame_manager(kRefFrameTableSize, 7);
   GopStruct gop_struct =
       ConstructGop(&ref_frame_manager, show_frame_count, has_key_frame,
                    global_coding_idx_offset, global_order_idx_offset);
@@ -266,7 +266,7 @@
   const bool has_key_frame = true;
   const int global_coding_idx_offset = 10;
   const int global_order_idx_offset = 8;
-  RefFrameManager ref_frame_manager(kRefFrameTableSize);
+  RefFrameManager ref_frame_manager(kRefFrameTableSize, 7);
   GopStruct gop_struct =
       ConstructGop(&ref_frame_manager, show_frame_count, has_key_frame,
                    global_coding_idx_offset, global_order_idx_offset);
@@ -286,7 +286,7 @@
   const bool has_key_frame = false;
   const int global_coding_idx_offset = 5;
   const int global_order_idx_offset = 20;
-  RefFrameManager ref_frame_manager(kRefFrameTableSize);
+  RefFrameManager ref_frame_manager(kRefFrameTableSize, 7);
   GopStruct gop_struct =
       ConstructGop(&ref_frame_manager, show_frame_count, has_key_frame,
                    global_coding_idx_offset, global_order_idx_offset);
@@ -630,7 +630,7 @@
     GopFrameType::kRegularLeaf,
     GopFrameType::kOverlay
   };
-  RefFrameManager ref_manager(kRefFrameTableSize);
+  RefFrameManager ref_manager(kRefFrameTableSize, 7);
   int coding_idx = 0;
   const int first_leaf_idx = 3;
   EXPECT_EQ(type_list[first_leaf_idx], GopFrameType::kRegularLeaf);
@@ -731,7 +731,7 @@
     GopFrameType::kRegularLeaf,
     GopFrameType::kOverlay
   };
-  RefFrameManager ref_manager(kRefFrameTableSize);
+  RefFrameManager ref_manager(kRefFrameTableSize, 7);
   int coding_idx = 0;
   const int first_leaf_idx = 3;
   EXPECT_EQ(type_list[first_leaf_idx], GopFrameType::kRegularLeaf);
@@ -767,7 +767,7 @@
                                                 GopFrameType::kRegularArf,
                                                 GopFrameType::kIntermediateArf,
                                                 GopFrameType::kRegularLeaf };
-  RefFrameManager ref_manager(kRefFrameTableSize);
+  RefFrameManager ref_manager(kRefFrameTableSize, 7);
   for (int coding_idx = 0; coding_idx < frame_count; ++coding_idx) {
     GopFrame gop_frame =
         GopFrameBasic(0, 0, coding_idx, order_idx_list[coding_idx], 0, 0,
@@ -802,7 +802,7 @@
                                                 GopFrameType::kIntermediateArf,
                                                 GopFrameType::kRegularLeaf };
   const std::vector<int> layer_depth_list = { 0, 2, 4, 6 };
-  RefFrameManager ref_manager(kRefFrameTableSize);
+  RefFrameManager ref_manager(kRefFrameTableSize, 7);
   for (int coding_idx = 0; coding_idx < frame_count; ++coding_idx) {
     GopFrame gop_frame =
         GopFrameBasic(0, 0, coding_idx, order_idx_list[coding_idx],
@@ -816,6 +816,7 @@
     // Set different frame type
     GopFrameType type = type_list[(i + 1) % frame_count];
     GopFrame gop_frame = GopFrameBasic(0, 0, 0, 0, layer_depth, 0, type);
+    gop_frame.ref_frame_list = ref_manager.GetRefFrameListByPriority();
     ReferenceFrame ref_frame = ref_manager.GetPrimaryRefFrame(gop_frame);
     GopFrame primary_ref_frame =
         ref_manager.GetRefFrameByIndex(ref_frame.index);
@@ -831,6 +832,7 @@
     // Let the frame layer_depth sit in the middle of two reference frames
     int layer_depth = mid_layer_depth_list[i];
     GopFrame gop_frame = GopFrameBasic(0, 0, 0, 0, layer_depth, 0, type);
+    gop_frame.ref_frame_list = ref_manager.GetRefFrameListByPriority();
     ReferenceFrame ref_frame = ref_manager.GetPrimaryRefFrame(gop_frame);
     GopFrame primary_ref_frame =
         ref_manager.GetRefFrameByIndex(ref_frame.index);