diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 3e8bfcb..6048a2c 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -432,7 +432,7 @@
   ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
 
 typedef struct RefBuffer {
-  int idx;      // frame buf idx
+  int idx;      // Index into 'cm->buffer_pool->frame_bufs'.
   int map_idx;  // frame map idx
   YV12_BUFFER_CONFIG *buf;
   struct scale_factors sf;
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index cc34dd6..551e971 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -279,22 +279,28 @@
   // TODO(hkuang): Combine this with cur_buf in macroblockd.
   RefCntBuffer *cur_frame;
 
-  int ref_frame_map[REF_FRAMES]; /* maps fb_idx to reference slot */
+  // For decoder, ref_frame_map[i] maps reference type 'i' to actual index of
+  // the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
+  // For encoder, ref_frame_map[j] (where j = remapped_ref_idx[i]) maps
+  // remapped reference index 'j' (that is, original reference type 'i') to
+  // actual index of the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
+  int ref_frame_map[REF_FRAMES];
 
   // Prepare ref_frame_map for the next frame.
   // Only used in frame parallel decode.
   int next_ref_frame_map[REF_FRAMES];
 
-  // TODO(jkoleszar): could expand active_ref_idx to 4, with 0 as intra, and
-  // roll new_fb_idx into it.
-
-  // Each Inter frame can reference INTER_REFS_PER_FRAME buffers
+  // Each Inter frame can reference INTER_REFS_PER_FRAME buffers. This maps each
+  // (inter) reference frame type to the corresponding reference buffer.
   RefBuffer frame_refs[INTER_REFS_PER_FRAME];
+
   int is_skip_mode_allowed;
   int skip_mode_flag;
   int ref_frame_idx_0;
   int ref_frame_idx_1;
 
+  // Index to the 'new' frame (i.e. the frame currently being encoded or
+  // decoded) in the buffer pool 'cm->buffer_pool'.
   int new_fb_idx;
 
   FRAME_TYPE last_frame_type; /* last frame's frame type for motion search.*/
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 1c53a2e..2c779ed 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -2222,18 +2222,19 @@
   //     shifted and become the new virtual indexes for LAST2_FRAME and
   //     LAST3_FRAME.
   refresh_mask |=
-      (cpi->refresh_last_frame << cpi->ref_fb_idx[LAST_REF_FRAMES - 1]);
+      (cpi->refresh_last_frame << cpi->remapped_ref_idx[LAST_REF_FRAMES - 1]);
 #if USE_SYMM_MULTI_LAYER
-  refresh_mask |=
-      (cpi->new_bwdref_update_rule == 1)
-          ? (cpi->refresh_bwd_ref_frame << cpi->ref_fb_idx[EXTREF_FRAME - 1])
-          : (cpi->refresh_bwd_ref_frame << cpi->ref_fb_idx[BWDREF_FRAME - 1]);
+  refresh_mask |= (cpi->new_bwdref_update_rule == 1)
+                      ? (cpi->refresh_bwd_ref_frame
+                         << cpi->remapped_ref_idx[EXTREF_FRAME - 1])
+                      : (cpi->refresh_bwd_ref_frame
+                         << cpi->remapped_ref_idx[BWDREF_FRAME - 1]);
 #else
   refresh_mask |=
-      (cpi->refresh_bwd_ref_frame << cpi->ref_fb_idx[BWDREF_FRAME - 1]);
+      (cpi->refresh_bwd_ref_frame << cpi->remapped_ref_idx[BWDREF_FRAME - 1]);
 #endif
   refresh_mask |=
-      (cpi->refresh_alt2_ref_frame << cpi->ref_fb_idx[ALTREF2_FRAME - 1]);
+      (cpi->refresh_alt2_ref_frame << cpi->remapped_ref_idx[ALTREF2_FRAME - 1]);
 
   if (av1_preserve_existing_gf(cpi)) {
     // We have decided to preserve the previously existing golden frame as our
@@ -2250,13 +2251,14 @@
     if (cpi->preserve_arf_as_gld) {
       return refresh_mask;
     } else {
-      return refresh_mask |
-             (cpi->refresh_golden_frame << cpi->ref_fb_idx[ALTREF_FRAME - 1]);
+      return refresh_mask | (cpi->refresh_golden_frame
+                             << cpi->remapped_ref_idx[ALTREF_FRAME - 1]);
     }
   } else {
-    const int arf_idx = cpi->ref_fb_idx[ALTREF_FRAME - 1];
+    const int arf_idx = cpi->remapped_ref_idx[ALTREF_FRAME - 1];
     return refresh_mask |
-           (cpi->refresh_golden_frame << cpi->ref_fb_idx[GOLDEN_FRAME - 1]) |
+           (cpi->refresh_golden_frame
+            << cpi->remapped_ref_idx[GOLDEN_FRAME - 1]) |
            (cpi->refresh_alt_ref_frame << arf_idx);
   }
 }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 95d47fa..b108111 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -917,7 +917,7 @@
 static void init_buffer_indices(AV1_COMP *cpi) {
   int fb_idx;
   for (fb_idx = 0; fb_idx < REF_FRAMES; ++fb_idx)
-    cpi->ref_fb_idx[fb_idx] = fb_idx;
+    cpi->remapped_ref_idx[fb_idx] = fb_idx;
   cpi->rate_index = 0;
   cpi->rate_size = 0;
   cpi->cur_poc = -1;
@@ -2558,7 +2558,6 @@
   cpi->partition_search_skippable_frame = 0;
   cpi->tile_data = NULL;
   cpi->last_show_frame_buf_idx = INVALID_IDX;
-
   realloc_segmentation_maps(cpi);
 
   memset(cpi->nmv_costs, 0, sizeof(cpi->nmv_costs));
@@ -3231,7 +3230,7 @@
       //       the last_fb_idxes[0] after reference frame buffer update
       cpi->rc.is_last_bipred_frame = 0;
       cm->show_existing_frame = 1;
-      cpi->existing_fb_idx_to_show = cpi->ref_fb_idx[0];
+      cpi->existing_fb_idx_to_show = cpi->remapped_ref_idx[0];
 #if USE_SYMM_MULTI_LAYER
     }
 #endif
@@ -3248,9 +3247,10 @@
     // in av1_rc_get_second_pass_params(cpi)
     cm->show_existing_frame = 1;
     cpi->rc.is_src_frame_alt_ref = 1;
-    cpi->existing_fb_idx_to_show = (next_frame_update_type == OVERLAY_UPDATE)
-                                       ? cpi->ref_fb_idx[ALTREF_FRAME - 1]
-                                       : cpi->ref_fb_idx[bwdref_to_show - 1];
+    cpi->existing_fb_idx_to_show =
+        (next_frame_update_type == OVERLAY_UPDATE)
+            ? cpi->remapped_ref_idx[ALTREF_FRAME - 1]
+            : cpi->remapped_ref_idx[bwdref_to_show - 1];
 #if USE_SYMM_MULTI_LAYER
     if (cpi->new_bwdref_update_rule == 0)
 #endif
@@ -3427,7 +3427,7 @@
   // TODO(isbs): shift the scaled indices as well
   int ref_frame;
   for (ref_frame = LAST_REF_FRAMES - 1; ref_frame > 0; --ref_frame) {
-    cpi->ref_fb_idx[ref_frame] = cpi->ref_fb_idx[ref_frame - 1];
+    cpi->remapped_ref_idx[ref_frame] = cpi->remapped_ref_idx[ref_frame - 1];
 
     // [0] is allocated to the current coded frame. The statistics for the
     // reference frames start at [LAST_FRAME], i.e. [1].
@@ -3456,7 +3456,8 @@
         cpi->interp_filter_selected[ordered_bwd[i - 1] + LAST_FRAME],
         sizeof(cpi->interp_filter_selected[ordered_bwd[i - 1] + LAST_FRAME]));
 
-    cpi->ref_fb_idx[ordered_bwd[i]] = cpi->ref_fb_idx[ordered_bwd[i - 1]];
+    cpi->remapped_ref_idx[ordered_bwd[i]] =
+        cpi->remapped_ref_idx[ordered_bwd[i - 1]];
   }
 }
 
@@ -3476,7 +3477,8 @@
         cpi->interp_filter_selected[ordered_bwd[i + 1] + LAST_FRAME],
         sizeof(cpi->interp_filter_selected[ordered_bwd[i + 1] + LAST_FRAME]));
 
-    cpi->ref_fb_idx[ordered_bwd[i]] = cpi->ref_fb_idx[ordered_bwd[i + 1]];
+    cpi->remapped_ref_idx[ordered_bwd[i]] =
+        cpi->remapped_ref_idx[ordered_bwd[i + 1]];
   }
 }
 #endif  // USE_SYMM_MULTI_LAYER
@@ -3518,7 +3520,7 @@
   if ((cm->frame_type == KEY_FRAME && cm->show_frame) || frame_is_sframe(cm)) {
     for (int ref_frame = 0; ref_frame < REF_FRAMES; ++ref_frame) {
       ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->ref_fb_idx[ref_frame]],
+                 &cm->ref_frame_map[cpi->remapped_ref_idx[ref_frame]],
                  cm->new_fb_idx);
     }
     return;
@@ -3540,13 +3542,14 @@
 
     if (!cpi->preserve_arf_as_gld) {
       ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->ref_fb_idx[ALTREF_FRAME - 1]],
+                 &cm->ref_frame_map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]],
                  cm->new_fb_idx);
     }
 
-    tmp = cpi->ref_fb_idx[ALTREF_FRAME - 1];
-    cpi->ref_fb_idx[ALTREF_FRAME - 1] = cpi->ref_fb_idx[GOLDEN_FRAME - 1];
-    cpi->ref_fb_idx[GOLDEN_FRAME - 1] = tmp;
+    tmp = cpi->remapped_ref_idx[ALTREF_FRAME - 1];
+    cpi->remapped_ref_idx[ALTREF_FRAME - 1] =
+        cpi->remapped_ref_idx[GOLDEN_FRAME - 1];
+    cpi->remapped_ref_idx[GOLDEN_FRAME - 1] = tmp;
 
     // TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to
     // cpi->interp_filter_selected[GOLDEN_FRAME]?
@@ -3564,10 +3567,11 @@
     // Deal with the special case for showing existing internal ALTREF_FRAME
     // Refresh the LAST_FRAME with the ALTREF_FRAME and retire the LAST3_FRAME
     // by updating the virtual indices.
-    const int tmp = cpi->ref_fb_idx[LAST_REF_FRAMES - 1];
+    const int tmp = cpi->remapped_ref_idx[LAST_REF_FRAMES - 1];
     shift_last_ref_frames(cpi);
 
-    cpi->ref_fb_idx[LAST_FRAME - 1] = cpi->ref_fb_idx[bwdref_to_show - 1];
+    cpi->remapped_ref_idx[LAST_FRAME - 1] =
+        cpi->remapped_ref_idx[bwdref_to_show - 1];
 
     memcpy(cpi->interp_filter_selected[LAST_FRAME],
            cpi->interp_filter_selected[bwdref_to_show],
@@ -3577,17 +3581,17 @@
       lshift_bwd_ref_frames(cpi);
       // pass outdated forward reference frame (previous LAST3) to the
       // spared space
-      cpi->ref_fb_idx[EXTREF_FRAME - 1] = tmp;
+      cpi->remapped_ref_idx[EXTREF_FRAME - 1] = tmp;
     } else {
 #endif
-      cpi->ref_fb_idx[bwdref_to_show - 1] = tmp;
+      cpi->remapped_ref_idx[bwdref_to_show - 1] = tmp;
 #if USE_SYMM_MULTI_LAYER
     }
 #endif
   } else { /* For non key/golden frames */
     // === ALTREF_FRAME ===
     if (cpi->refresh_alt_ref_frame) {
-      int arf_idx = cpi->ref_fb_idx[ALTREF_FRAME - 1];
+      int arf_idx = cpi->remapped_ref_idx[ALTREF_FRAME - 1];
       ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[arf_idx], cm->new_fb_idx);
 
       memcpy(cpi->interp_filter_selected[ALTREF_FRAME],
@@ -3598,7 +3602,7 @@
     // === GOLDEN_FRAME ===
     if (cpi->refresh_golden_frame) {
       ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]],
+                 &cm->ref_frame_map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]],
                  cm->new_fb_idx);
 
       memcpy(cpi->interp_filter_selected[GOLDEN_FRAME],
@@ -3614,15 +3618,15 @@
         // BWDREF -> ALTREF2 -> EXTREF
         // and assign the newly coded frame to BWDREF so that it always
         // keeps the nearest future frame
-        int tmp = cpi->ref_fb_idx[EXTREF_FRAME - 1];
+        int tmp = cpi->remapped_ref_idx[EXTREF_FRAME - 1];
         ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[tmp], cm->new_fb_idx);
 
         rshift_bwd_ref_frames(cpi);
-        cpi->ref_fb_idx[BWDREF_FRAME - 1] = tmp;
+        cpi->remapped_ref_idx[BWDREF_FRAME - 1] = tmp;
       } else {
 #endif  // USE_SYMM_MULTI_LAYER
         ref_cnt_fb(pool->frame_bufs,
-                   &cm->ref_frame_map[cpi->ref_fb_idx[BWDREF_FRAME - 1]],
+                   &cm->ref_frame_map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]],
                    cm->new_fb_idx);
 #if USE_SYMM_MULTI_LAYER
       }
@@ -3635,7 +3639,7 @@
     // === ALTREF2_FRAME ===
     if (cpi->refresh_alt2_ref_frame) {
       ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]],
+                 &cm->ref_frame_map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]],
                  cm->new_fb_idx);
 
       memcpy(cpi->interp_filter_selected[ALTREF2_FRAME],
@@ -3649,13 +3653,13 @@
     // reference to the reference frame buffer virtual index; and then (2) from
     // the virtual index to the reference frame buffer physical index:
     //
-    // LAST_FRAME,      ..., LAST3_FRAME,     ..., ALTREF_FRAME
-    //      |                     |                     |
-    //      v                     v                     v
-    // ref_fb_idx[0],   ..., ref_fb_idx[2],   ..., ref_fb_idx[ALTREF_FRAME-1]
-    //      |                     |                     |
-    //      v                     v                     v
-    // ref_frame_map[], ..., ref_frame_map[], ..., ref_frame_map[]
+    // LAST_FRAME,                       ...,     EXTREF_FRAME
+    //      |                                           |
+    //      v                                           v
+    // remapped_ref_idx[LAST_FRAME - 1], ..., remapped_ref_idx[EXTREF_FRAME - 1]
+    //      |                                           |
+    //      v                                           v
+    // ref_frame_map[],                  ...,    ref_frame_map[]
     //
     // When refresh_last_frame is set, it is intended to retire LAST3_FRAME,
     // have the other 2 LAST reference frames shifted as follows:
@@ -3671,20 +3675,20 @@
     // (b) To change the 1st layer mapping to have LAST_FRAME mapped to the
     //     original virtual index of LAST3_FRAME and have the other mappings
     //     shifted as follows:
-    // LAST_FRAME,      LAST2_FRAME,     LAST3_FRAME
-    //      |                |                |
-    //      v                v                v
-    // ref_fb_idx[2],   ref_fb_idx[0],   ref_fb_idx[1]
+    // LAST_FRAME,            LAST2_FRAME,             LAST3_FRAME
+    //      |                      |                        |
+    //      v                      v                        v
+    // remapped_ref_idx[2],   remapped_ref_idx[0],     remapped_ref_idx[1]
     int tmp;
 
     ref_cnt_fb(pool->frame_bufs,
-               &cm->ref_frame_map[cpi->ref_fb_idx[LAST_REF_FRAMES - 1]],
+               &cm->ref_frame_map[cpi->remapped_ref_idx[LAST_REF_FRAMES - 1]],
                cm->new_fb_idx);
 
-    tmp = cpi->ref_fb_idx[LAST_REF_FRAMES - 1];
+    tmp = cpi->remapped_ref_idx[LAST_REF_FRAMES - 1];
 
     shift_last_ref_frames(cpi);
-    cpi->ref_fb_idx[0] = tmp;
+    cpi->remapped_ref_idx[0] = tmp;
 
     assert(!encode_show_existing_frame(cm));
     memcpy(cpi->interp_filter_selected[LAST_FRAME],
@@ -3707,11 +3711,11 @@
       //       virtual index reshuffling for BWDREF, the encoder always
       //       specifies a LAST_BIPRED right before BWDREF and completes the
       //       reshuffling job accordingly.
-      tmp = cpi->ref_fb_idx[LAST_REF_FRAMES - 1];
+      tmp = cpi->remapped_ref_idx[LAST_REF_FRAMES - 1];
 
       shift_last_ref_frames(cpi);
-      cpi->ref_fb_idx[0] = cpi->ref_fb_idx[BWDREF_FRAME - 1];
-      cpi->ref_fb_idx[BWDREF_FRAME - 1] = tmp;
+      cpi->remapped_ref_idx[0] = cpi->remapped_ref_idx[BWDREF_FRAME - 1];
+      cpi->remapped_ref_idx[BWDREF_FRAME - 1] = tmp;
 
       memcpy(cpi->interp_filter_selected[LAST_FRAME],
              cpi->interp_filter_selected[BWDREF_FRAME],
@@ -4721,55 +4725,58 @@
   const int *const map = cpi->common.ref_frame_map;
 
   // No.1 Priority: LAST_FRAME
-  const int last2_is_last = map[cpi->ref_fb_idx[1]] == map[cpi->ref_fb_idx[0]];
-  const int last3_is_last = map[cpi->ref_fb_idx[2]] == map[cpi->ref_fb_idx[0]];
-  const int gld_is_last =
-      map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == map[cpi->ref_fb_idx[0]];
-  const int bwd_is_last =
-      map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == map[cpi->ref_fb_idx[0]];
-  const int alt2_is_last =
-      map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == map[cpi->ref_fb_idx[0]];
-  const int alt_is_last =
-      map[cpi->ref_fb_idx[ALTREF_FRAME - 1]] == map[cpi->ref_fb_idx[0]];
+  const int last2_is_last =
+      map[cpi->remapped_ref_idx[1]] == map[cpi->remapped_ref_idx[0]];
+  const int last3_is_last =
+      map[cpi->remapped_ref_idx[2]] == map[cpi->remapped_ref_idx[0]];
+  const int gld_is_last = map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]] ==
+                          map[cpi->remapped_ref_idx[0]];
+  const int bwd_is_last = map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]] ==
+                          map[cpi->remapped_ref_idx[0]];
+  const int alt2_is_last = map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]] ==
+                           map[cpi->remapped_ref_idx[0]];
+  const int alt_is_last = map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]] ==
+                          map[cpi->remapped_ref_idx[0]];
 
   // No.2 Priority: ALTREF_FRAME
-  const int last2_is_alt =
-      map[cpi->ref_fb_idx[1]] == map[cpi->ref_fb_idx[ALTREF_FRAME - 1]];
-  const int last3_is_alt =
-      map[cpi->ref_fb_idx[2]] == map[cpi->ref_fb_idx[ALTREF_FRAME - 1]];
-  const int gld_is_alt = map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] ==
-                         map[cpi->ref_fb_idx[ALTREF_FRAME - 1]];
-  const int bwd_is_alt = map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] ==
-                         map[cpi->ref_fb_idx[ALTREF_FRAME - 1]];
-  const int alt2_is_alt = map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] ==
-                          map[cpi->ref_fb_idx[ALTREF_FRAME - 1]];
+  const int last2_is_alt = map[cpi->remapped_ref_idx[1]] ==
+                           map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]];
+  const int last3_is_alt = map[cpi->remapped_ref_idx[2]] ==
+                           map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]];
+  const int gld_is_alt = map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]] ==
+                         map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]];
+  const int bwd_is_alt = map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]] ==
+                         map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]];
+  const int alt2_is_alt = map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]] ==
+                          map[cpi->remapped_ref_idx[ALTREF_FRAME - 1]];
 
   // No.3 Priority: LAST2_FRAME
-  const int last3_is_last2 = map[cpi->ref_fb_idx[2]] == map[cpi->ref_fb_idx[1]];
-  const int gld_is_last2 =
-      map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == map[cpi->ref_fb_idx[1]];
-  const int bwd_is_last2 =
-      map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == map[cpi->ref_fb_idx[1]];
-  const int alt2_is_last2 =
-      map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == map[cpi->ref_fb_idx[1]];
+  const int last3_is_last2 =
+      map[cpi->remapped_ref_idx[2]] == map[cpi->remapped_ref_idx[1]];
+  const int gld_is_last2 = map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]] ==
+                           map[cpi->remapped_ref_idx[1]];
+  const int bwd_is_last2 = map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]] ==
+                           map[cpi->remapped_ref_idx[1]];
+  const int alt2_is_last2 = map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]] ==
+                            map[cpi->remapped_ref_idx[1]];
 
   // No.4 Priority: LAST3_FRAME
-  const int gld_is_last3 =
-      map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]] == map[cpi->ref_fb_idx[2]];
-  const int bwd_is_last3 =
-      map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] == map[cpi->ref_fb_idx[2]];
-  const int alt2_is_last3 =
-      map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] == map[cpi->ref_fb_idx[2]];
+  const int gld_is_last3 = map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]] ==
+                           map[cpi->remapped_ref_idx[2]];
+  const int bwd_is_last3 = map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]] ==
+                           map[cpi->remapped_ref_idx[2]];
+  const int alt2_is_last3 = map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]] ==
+                            map[cpi->remapped_ref_idx[2]];
 
   // No.5 Priority: GOLDEN_FRAME
-  const int bwd_is_gld = map[cpi->ref_fb_idx[BWDREF_FRAME - 1]] ==
-                         map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]];
-  const int alt2_is_gld = map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] ==
-                          map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]];
+  const int bwd_is_gld = map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]] ==
+                         map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]];
+  const int alt2_is_gld = map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]] ==
+                          map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]];
 
   // No.6 Priority: BWDREF_FRAME
-  const int alt2_is_bwd = map[cpi->ref_fb_idx[ALTREF2_FRAME - 1]] ==
-                          map[cpi->ref_fb_idx[BWDREF_FRAME - 1]];
+  const int alt2_is_bwd = map[cpi->remapped_ref_idx[ALTREF2_FRAME - 1]] ==
+                          map[cpi->remapped_ref_idx[BWDREF_FRAME - 1]];
 
   // No.7 Priority: ALTREF2_FRAME
 
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 42c477b..fcf9f2a 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -622,8 +622,25 @@
 
   unsigned int row_mt;
   int scaled_ref_idx[INTER_REFS_PER_FRAME];
-  int ref_fb_idx[REF_FRAMES];
-  int refresh_fb_idx;  // ref frame buffer index to refresh
+
+  // For encoder, we have a two-level mapping from reference frame type to the
+  // corresponding buffer in the buffer pool:
+  // * 'remapped_ref_idx[i - 1]' maps reference type ‘i’ (range: LAST_FRAME ...
+  // EXTREF_FRAME) to a remapped index ‘j’ (in range: 0 ... REF_FRAMES - 1)
+  // * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to actual index
+  //   of the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
+  //
+  // LAST_FRAME,                        ...,      EXTREF_FRAME
+  //      |                                           |
+  //      v                                           v
+  // remapped_ref_idx[LAST_FRAME - 1],  ...,  remapped_ref_idx[EXTREF_FRAME - 1]
+  //      |                                           |
+  //      v                                           v
+  // ref_frame_map[],                   ...,     ref_frame_map[]
+  //
+  // Note: INTRA_FRAME always refers to the current frame, so there's no need to
+  // have a remapped index for the same.
+  int remapped_ref_idx[REF_FRAMES];
 
   int last_show_frame_buf_idx;  // last show frame buffer index
 
@@ -909,7 +926,7 @@
 
 static INLINE int get_ref_frame_map_idx(const AV1_COMP *cpi,
                                         MV_REFERENCE_FRAME ref_frame) {
-  return (ref_frame >= 1) ? cpi->ref_fb_idx[ref_frame - 1] : INVALID_IDX;
+  return (ref_frame >= 1) ? cpi->remapped_ref_idx[ref_frame - 1] : INVALID_IDX;
 }
 
 static INLINE int get_ref_frame_buf_idx(const AV1_COMP *cpi,
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index 1098530..00aa0ad 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -1061,8 +1061,8 @@
          DOUBLE_DIVIDE_CHECK(twopass->this_frame_stats.coded_error)) > 2.0))) {
     if (gld_yv12 != NULL) {
       ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]],
-                 cm->ref_frame_map[cpi->ref_fb_idx[LAST_FRAME - 1]]);
+                 &cm->ref_frame_map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]],
+                 cm->ref_frame_map[cpi->remapped_ref_idx[LAST_FRAME - 1]]);
     }
     twopass->sr_update_lag = 1;
   } else {
@@ -1073,16 +1073,16 @@
 
   // The frame we just compressed now becomes the last frame.
   ref_cnt_fb(pool->frame_bufs,
-             &cm->ref_frame_map[cpi->ref_fb_idx[LAST_FRAME - 1]],
+             &cm->ref_frame_map[cpi->remapped_ref_idx[LAST_FRAME - 1]],
              cm->new_fb_idx);
 
   // Special case for the first frame. Copy into the GF buffer as a second
   // reference.
   if (cm->current_video_frame == 0 &&
-      cpi->ref_fb_idx[GOLDEN_FRAME - 1] != INVALID_IDX) {
+      cpi->remapped_ref_idx[GOLDEN_FRAME - 1] != INVALID_IDX) {
     ref_cnt_fb(pool->frame_bufs,
-               &cm->ref_frame_map[cpi->ref_fb_idx[GOLDEN_FRAME - 1]],
-               cm->ref_frame_map[cpi->ref_fb_idx[LAST_FRAME - 1]]);
+               &cm->ref_frame_map[cpi->remapped_ref_idx[GOLDEN_FRAME - 1]],
+               cm->ref_frame_map[cpi->remapped_ref_idx[LAST_FRAME - 1]]);
   }
 
   // Use this to see what the first pass reconstruction looks like.