diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 08f3f2b..4a1827d 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -3444,18 +3444,17 @@
 // to clear a space to store the closest bwdref
 static INLINE void rshift_bwd_ref_frames(AV1_COMP *cpi) {
   // TODO(isbs): shift the scaled indices as well
-  static const int ordered_bwd[3] = { BWDREF_FRAME - 1, ALTREF2_FRAME - 1,
-                                      EXTREF_FRAME - 1 };
+  static const int ordered_bwd[3] = { BWDREF_FRAME, ALTREF2_FRAME,
+                                      EXTREF_FRAME };
 
   for (int i = 2; i > 0; --i) {
     // [0] is allocated to the current coded frame, i.e. bwdref
-    memcpy(
-        cpi->interp_filter_selected[ordered_bwd[i] + LAST_FRAME],
-        cpi->interp_filter_selected[ordered_bwd[i - 1] + LAST_FRAME],
-        sizeof(cpi->interp_filter_selected[ordered_bwd[i - 1] + LAST_FRAME]));
+    memcpy(cpi->interp_filter_selected[ordered_bwd[i]],
+           cpi->interp_filter_selected[ordered_bwd[i - 1]],
+           sizeof(cpi->interp_filter_selected[ordered_bwd[i - 1]]));
 
-    cpi->remapped_ref_idx[ordered_bwd[i]] =
-        cpi->remapped_ref_idx[ordered_bwd[i - 1]];
+    cpi->remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] =
+        cpi->remapped_ref_idx[ordered_bwd[i - 1] - LAST_FRAME];
   }
 }
 
@@ -3465,18 +3464,17 @@
 // to update the bwd reference frame for coding the next frame.
 static INLINE void lshift_bwd_ref_frames(AV1_COMP *cpi) {
   // TODO(isbs): shift the scaled indices as well
-  static const int ordered_bwd[3] = { BWDREF_FRAME - 1, ALTREF2_FRAME - 1,
-                                      EXTREF_FRAME - 1 };
+  static const int ordered_bwd[3] = { BWDREF_FRAME, ALTREF2_FRAME,
+                                      EXTREF_FRAME };
 
   for (int i = 0; i < 2; ++i) {
     // [0] is allocated to the current coded frame, i.e. bwdref
-    memcpy(
-        cpi->interp_filter_selected[ordered_bwd[i] + LAST_FRAME],
-        cpi->interp_filter_selected[ordered_bwd[i + 1] + LAST_FRAME],
-        sizeof(cpi->interp_filter_selected[ordered_bwd[i + 1] + LAST_FRAME]));
+    memcpy(cpi->interp_filter_selected[ordered_bwd[i]],
+           cpi->interp_filter_selected[ordered_bwd[i + 1]],
+           sizeof(cpi->interp_filter_selected[ordered_bwd[i + 1]]));
 
-    cpi->remapped_ref_idx[ordered_bwd[i]] =
-        cpi->remapped_ref_idx[ordered_bwd[i + 1]];
+    cpi->remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] =
+        cpi->remapped_ref_idx[ordered_bwd[i + 1] - LAST_FRAME];
   }
 }
 #endif  // USE_SYMM_MULTI_LAYER
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index c7afcb7..cb286eb 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -709,8 +709,8 @@
 
   // Relevant for an inter frame.
   // - Index '0' corresponds to the values for the currently coded frame.
-  // - Indices 1 ... REF_FRAMES are used to store values for all the possible
-  // reference frames.
+  // - Indices LAST_FRAME ... EXTREF_FRAMES are used to store values for all the
+  // possible inter reference frames.
   int interp_filter_selected[REF_FRAMES + 1][SWITCHABLE];
 
   struct aom_codec_pkt_list *output_pkt_list;