diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index 0f4990e..bdcd4ff 100644
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -291,7 +291,7 @@
     const int ref_num = 0;
     const int is_intrabc = is_intrabc_block(mi);
     const struct scale_factors *const sf =
-        is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
+        is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
     const int is_scaled = av1_is_scaled(sf);
 
     if (is_scaled) {
@@ -880,7 +880,7 @@
     const int ref_num = 0;
     const int is_intrabc = is_intrabc_block(mi);
     const struct scale_factors *const sf =
-        is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
+        is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
     const int is_scaled = av1_is_scaled(sf);
 
     if (is_scaled) {
diff --git a/aom_dsp/x86/highbd_variance_sse2.c b/aom_dsp/x86/highbd_variance_sse2.c
index 226576b..07fcad4 100644
--- a/aom_dsp/x86/highbd_variance_sse2.c
+++ b/aom_dsp/x86/highbd_variance_sse2.c
@@ -603,7 +603,7 @@
     const int ref_num = 0;
     const int is_intrabc = is_intrabc_block(mi);
     const struct scale_factors *const sf =
-        is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
+        is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
     const int is_scaled = av1_is_scaled(sf);
 
     if (is_scaled) {
diff --git a/aom_dsp/x86/variance_sse2.c b/aom_dsp/x86/variance_sse2.c
index c831e3e..f3efc15 100644
--- a/aom_dsp/x86/variance_sse2.c
+++ b/aom_dsp/x86/variance_sse2.c
@@ -494,7 +494,7 @@
     const int ref_num = 0;
     const int is_intrabc = is_intrabc_block(mi);
     const struct scale_factors *const sf =
-        is_intrabc ? &cm->sf_identity : &xd->block_refs[ref_num]->sf;
+        is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref_num];
     const int is_scaled = av1_is_scaled(sf);
 
     if (is_scaled) {
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 14dfd80..7e79b5f 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -428,14 +428,6 @@
 #define BLOCK_OFFSET(x, i) \
   ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
 
-struct RefCntBuffer;
-
-typedef struct RefBuffer {
-  int map_idx;  // frame map idx
-  struct RefCntBuffer *buf;
-  struct scale_factors sf;
-} RefBuffer;
-
 typedef struct {
   DECLARE_ALIGNED(16, InterpKernel, vfilter);
   DECLARE_ALIGNED(16, InterpKernel, hfilter);
@@ -497,6 +489,8 @@
   int bck_offset;
 } JNT_COMP_PARAMS;
 
+struct scale_factors;
+
 // Most/all of the pointers are mere pointers to actual arrays are allocated
 // elsewhere. This is mostly for coding convenience.
 typedef struct macroblockd {
@@ -523,8 +517,8 @@
   int mb_to_top_edge;
   int mb_to_bottom_edge;
 
-  /* pointers to reference frames */
-  const RefBuffer *block_refs[2];
+  /* pointers to reference frame scale factors */
+  const struct scale_factors *block_ref_scale_factors[2];
 
   /* pointer to current frame */
   const YV12_BUFFER_CONFIG *cur_buf;
@@ -1046,7 +1040,8 @@
     if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION;
     assert(!has_second_ref(mbmi));
     if (mbmi->num_proj_ref >= 1 &&
-        (allow_warped_motion && !av1_is_scaled(&(xd->block_refs[0]->sf)))) {
+        (allow_warped_motion &&
+         !av1_is_scaled(xd->block_ref_scale_factors[0]))) {
       if (xd->cur_frame_force_integer_mv) {
         return OBMC_CAUSAL;
       }
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 3a3764d..8e7e952 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1072,9 +1072,9 @@
   // TODO(jack.haughton@argondesign.com): don't think this should be necessary,
   // but could do with fuller testing
   if (cm->large_scale_tile) {
-    for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-      if (cm->current_frame.frame_refs[i].buf != NULL)
-        cm->current_frame.frame_refs[i].buf->frame_context = *cm->fc;
+    for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
+      RefCntBuffer *const buf = get_ref_frame_buf(cm, i);
+      if (buf != NULL) buf->frame_context = *cm->fc;
     }
     for (int i = 0; i < FRAME_BUFFERS; ++i)
       cm->buffer_pool->frame_bufs[i].frame_context = *cm->fc;
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c
index 5bf0cb5..60cb3d1 100644
--- a/av1/common/mvref_common.c
+++ b/av1/common/mvref_common.c
@@ -347,8 +347,7 @@
 
   if (rf[1] == NONE_FRAME) {
     int cur_frame_index = cm->cur_frame->order_hint;
-    const RefCntBuffer *const buf_0 =
-        cm->current_frame.frame_refs[FWD_RF_OFFSET(rf[0])].buf;
+    const RefCntBuffer *const buf_0 = get_ref_frame_buf(cm, rf[0]);
     int frame0_index = buf_0->order_hint;
     int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info,
                                          cur_frame_index, frame0_index);
@@ -383,14 +382,12 @@
   } else {
     // Process compound inter mode
     int cur_frame_index = cm->cur_frame->order_hint;
-    const RefCntBuffer *const buf_0 =
-        cm->current_frame.frame_refs[FWD_RF_OFFSET(rf[0])].buf;
+    const RefCntBuffer *const buf_0 = get_ref_frame_buf(cm, rf[0]);
     int frame0_index = buf_0->order_hint;
 
     int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info,
                                          cur_frame_index, frame0_index);
-    const RefCntBuffer *const buf_1 =
-        cm->current_frame.frame_refs[FWD_RF_OFFSET(rf[1])].buf;
+    const RefCntBuffer *const buf_1 = get_ref_frame_buf(cm, rf[1]);
     int frame1_index = buf_1->order_hint;
     int cur_offset_1 = get_relative_dist(&cm->seq_params.order_hint_info,
                                          cur_frame_index, frame1_index);
@@ -871,8 +868,7 @@
 
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const RefCntBuffer *const buf =
-        cm->current_frame.frame_refs[ref_frame - LAST_FRAME].buf;
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     if (buf != NULL)
       cm->cur_frame->ref_order_hints[ref_frame - LAST_FRAME] = buf->order_hint;
   }
@@ -881,8 +877,7 @@
 void av1_setup_frame_sign_bias(AV1_COMMON *cm) {
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const RefCntBuffer *const buf =
-        cm->current_frame.frame_refs[ref_frame - LAST_FRAME].buf;
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     if (cm->seq_params.order_hint_info.enable_order_hint && buf != NULL) {
       const int ref_order_hint = buf->order_hint;
       cm->ref_frame_sign_bias[ref_frame] =
@@ -945,7 +940,7 @@
   (void)dir;
 
   const RefCntBuffer *const start_frame_buf =
-      cm->current_frame.frame_refs[FWD_RF_OFFSET(start_frame)].buf;
+      get_ref_frame_buf(cm, start_frame);
   if (start_frame_buf == NULL) return 0;
 
   if (start_frame_buf->frame_type == KEY_FRAME ||
@@ -1031,7 +1026,7 @@
 
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
     const int ref_idx = ref_frame - LAST_FRAME;
-    const RefCntBuffer *const buf = cm->current_frame.frame_refs[ref_idx].buf;
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     int order_hint = 0;
 
     if (buf != NULL) order_hint = buf->order_hint;
@@ -1295,7 +1290,7 @@
 
   // Identify the nearest forward and backward references.
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    const RefCntBuffer *const buf = cm->current_frame.frame_refs[i].buf;
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, LAST_FRAME + i);
     if (buf == NULL) continue;
 
     const int ref_order_hint = buf->order_hint;
@@ -1330,7 +1325,7 @@
     // Identify the second nearest forward reference.
     ref_order_hints[1] = -1;
     for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-      const RefCntBuffer *const buf = cm->current_frame.frame_refs[i].buf;
+      const RefCntBuffer *const buf = get_ref_frame_buf(cm, LAST_FRAME + i);
       if (buf == NULL) continue;
 
       const int ref_order_hint = buf->order_hint;
@@ -1374,8 +1369,7 @@
                                REF_FRAME_INFO *ref_info) {
   assert(frame_idx >= 0 && frame_idx < INTER_REFS_PER_FRAME);
 
-  cm->current_frame.frame_refs[frame_idx].buf = ref_info->buf;
-  cm->current_frame.frame_refs[frame_idx].map_idx = ref_info->map_idx;
+  cm->remapped_ref_idx[frame_idx] = ref_info->map_idx;
 }
 
 void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index cc7d8db..d399463 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -315,9 +315,6 @@
   unsigned int order_hint;
   unsigned int frame_number;
   SkipModeInfo skip_mode_info;
-  // Each Inter frame can reference INTER_REFS_PER_FRAME buffers. This maps each
-  // (inter) reference frame type to the corresponding reference buffer.
-  RefBuffer frame_refs[INTER_REFS_PER_FRAME];
 } CurrentFrame;
 
 typedef struct AV1Common {
@@ -346,6 +343,28 @@
   // TODO(hkuang): Combine this with cur_buf in macroblockd.
   RefCntBuffer *cur_frame;
 
+  // For encoder, we have a two-level mapping from reference frame type to the
+  // corresponding buffer in the buffer pool:
+  // * 'remapped_ref_idx[i - 1]' maps reference type ‘i’ (range: LAST_FRAME ...
+  // EXTREF_FRAME) to a remapped index ‘j’ (in range: 0 ... REF_FRAMES - 1)
+  // * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to a pointer to
+  // the reference counted buffer structure RefCntBuffer, taken from the buffer
+  // pool cm->buffer_pool->frame_bufs.
+  //
+  // LAST_FRAME,                        ...,      EXTREF_FRAME
+  //      |                                           |
+  //      v                                           v
+  // remapped_ref_idx[LAST_FRAME - 1],  ...,  remapped_ref_idx[EXTREF_FRAME - 1]
+  //      |                                           |
+  //      v                                           v
+  // ref_frame_map[],                   ...,     ref_frame_map[]
+  //
+  // Note: INTRA_FRAME always refers to the current frame, so there's no need to
+  // have a remapped index for the same.
+  int remapped_ref_idx[REF_FRAMES];
+
+  struct scale_factors ref_scale_factors[REF_FRAMES];
+
   // For decoder, ref_frame_map[i] maps reference type 'i' to a pointer to
   // the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
   // For encoder, ref_frame_map[j] (where j = remapped_ref_idx[i]) maps
@@ -639,13 +658,41 @@
   return cm->current_frame.frame_type == S_FRAME;
 }
 
-static INLINE RefCntBuffer *get_prev_frame(const AV1_COMMON *const cm) {
-  if (cm->primary_ref_frame == PRIMARY_REF_NONE ||
-      cm->current_frame.frame_refs[cm->primary_ref_frame].buf == NULL) {
-    return NULL;
-  } else {
-    return cm->current_frame.frame_refs[cm->primary_ref_frame].buf;
-  }
+// These functions take a reference frame label between LAST_FRAME and
+// EXTREF_FRAME inclusive.  Note that this is different to the indexing
+// previously used by the frame_refs[] array.
+static INLINE int get_ref_frame_map_idx(const AV1_COMMON *const cm,
+                                        const MV_REFERENCE_FRAME ref_frame) {
+  return (ref_frame >= LAST_FRAME && ref_frame <= EXTREF_FRAME)
+             ? cm->remapped_ref_idx[ref_frame - LAST_FRAME]
+             : INVALID_IDX;
+}
+
+static INLINE RefCntBuffer *get_ref_frame_buf(
+    const AV1_COMMON *const cm, const MV_REFERENCE_FRAME ref_frame) {
+  const int map_idx = get_ref_frame_map_idx(cm, ref_frame);
+  return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL;
+}
+
+// Both const and non-const versions of this function are provided so that it
+// can be used with a const AV1_COMMON if needed.
+static INLINE const struct scale_factors *get_ref_scale_factors_const(
+    const AV1_COMMON *const cm, const MV_REFERENCE_FRAME ref_frame) {
+  const int map_idx = get_ref_frame_map_idx(cm, ref_frame);
+  return (map_idx != INVALID_IDX) ? &cm->ref_scale_factors[map_idx] : NULL;
+}
+
+static INLINE struct scale_factors *get_ref_scale_factors(
+    AV1_COMMON *const cm, const MV_REFERENCE_FRAME ref_frame) {
+  const int map_idx = get_ref_frame_map_idx(cm, ref_frame);
+  return (map_idx != INVALID_IDX) ? &cm->ref_scale_factors[map_idx] : NULL;
+}
+
+static INLINE RefCntBuffer *get_primary_ref_frame_buf(
+    const AV1_COMMON *const cm) {
+  if (cm->primary_ref_frame == PRIMARY_REF_NONE) return NULL;
+  const int map_idx = get_ref_frame_map_idx(cm, cm->primary_ref_frame + 1);
+  return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL;
 }
 
 // Returns 1 if this frame might allow mvs from some reference frame.
diff --git a/av1/common/pred_common.h b/av1/common/pred_common.h
index 4ddb843..1d69ef0 100644
--- a/av1/common/pred_common.h
+++ b/av1/common/pred_common.h
@@ -90,10 +90,8 @@
 static INLINE int get_comp_index_context(const AV1_COMMON *cm,
                                          const MACROBLOCKD *xd) {
   MB_MODE_INFO *mbmi = xd->mi[0];
-  const RefCntBuffer *const bck_buf =
-      cm->current_frame.frame_refs[mbmi->ref_frame[0] - LAST_FRAME].buf;
-  const RefCntBuffer *const fwd_buf =
-      cm->current_frame.frame_refs[mbmi->ref_frame[1] - LAST_FRAME].buf;
+  const RefCntBuffer *const bck_buf = get_ref_frame_buf(cm, mbmi->ref_frame[0]);
+  const RefCntBuffer *const fwd_buf = get_ref_frame_buf(cm, mbmi->ref_frame[1]);
   int bck_frame_index = 0, fwd_frame_index = 0;
   int cur_frame_index = cm->cur_frame->order_hint;
 
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c
index f338e1b..4a74efc 100644
--- a/av1/common/reconinter.c
+++ b/av1/common/reconinter.c
@@ -636,10 +636,8 @@
   }
 
   *use_jnt_comp_avg = 1;
-  const RefCntBuffer *const bck_buf =
-      cm->current_frame.frame_refs[mbmi->ref_frame[0] - LAST_FRAME].buf;
-  const RefCntBuffer *const fwd_buf =
-      cm->current_frame.frame_refs[mbmi->ref_frame[1] - LAST_FRAME].buf;
+  const RefCntBuffer *const bck_buf = get_ref_frame_buf(cm, mbmi->ref_frame[0]);
+  const RefCntBuffer *const fwd_buf = get_ref_frame_buf(cm, mbmi->ref_frame[1]);
   const int cur_frame_index = cm->cur_frame->order_hint;
   int bck_frame_index = 0, fwd_frame_index = 0;
 
@@ -968,15 +966,15 @@
   for (int ref = 0; ref < num_refs; ++ref) {
     const MV_REFERENCE_FRAME frame = above_mbmi->ref_frame[ref];
 
-    const RefBuffer *const ref_buf =
-        &ctxt->cm->current_frame.frame_refs[frame - LAST_FRAME];
-
-    xd->block_refs[ref] = ref_buf;
-    if ((!av1_is_valid_scale(&ref_buf->sf)))
+    const RefCntBuffer *const ref_buf = get_ref_frame_buf(ctxt->cm, frame);
+    const struct scale_factors *const sf =
+        get_ref_scale_factors_const(ctxt->cm, frame);
+    xd->block_ref_scale_factors[ref] = sf;
+    if ((!av1_is_valid_scale(sf)))
       aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM,
                          "Reference frame has invalid dimensions");
-    av1_setup_pre_planes(xd, ref, &ref_buf->buf->buf, ctxt->mi_row,
-                         above_mi_col, &ref_buf->sf, num_planes);
+    av1_setup_pre_planes(xd, ref, &ref_buf->buf, ctxt->mi_row, above_mi_col, sf,
+                         num_planes);
   }
 
   xd->mb_to_left_edge = 8 * MI_SIZE * (-above_mi_col);
@@ -1006,15 +1004,16 @@
   for (int ref = 0; ref < num_refs; ++ref) {
     const MV_REFERENCE_FRAME frame = left_mbmi->ref_frame[ref];
 
-    const RefBuffer *const ref_buf =
-        &ctxt->cm->current_frame.frame_refs[frame - LAST_FRAME];
+    const RefCntBuffer *const ref_buf = get_ref_frame_buf(ctxt->cm, frame);
+    const struct scale_factors *const ref_scale_factors =
+        get_ref_scale_factors_const(ctxt->cm, frame);
 
-    xd->block_refs[ref] = ref_buf;
-    if ((!av1_is_valid_scale(&ref_buf->sf)))
+    xd->block_ref_scale_factors[ref] = ref_scale_factors;
+    if ((!av1_is_valid_scale(ref_scale_factors)))
       aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM,
                          "Reference frame has invalid dimensions");
-    av1_setup_pre_planes(xd, ref, &ref_buf->buf->buf, left_mi_row, ctxt->mi_col,
-                         &ref_buf->sf, num_planes);
+    av1_setup_pre_planes(xd, ref, &ref_buf->buf, left_mi_row, ctxt->mi_col,
+                         ref_scale_factors, num_planes);
   }
 
   xd->mb_to_top_edge = 8 * MI_SIZE * (-left_mi_row);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 2addfd9..802c2eb 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -701,22 +701,23 @@
         uint8_t *dst = dst_buf->buf + dst_buf->stride * y + x;
 
         ref = 0;
-        const RefBuffer *ref_buf =
-            &cm->current_frame
-                 .frame_refs[this_mbmi->ref_frame[ref] - LAST_FRAME];
+        const RefCntBuffer *ref_buf =
+            get_ref_frame_buf(cm, this_mbmi->ref_frame[ref]);
+        const struct scale_factors *ref_scale_factors =
+            get_ref_scale_factors_const(cm, this_mbmi->ref_frame[ref]);
 
-        pd->pre[ref].buf0 = (plane == 1) ? ref_buf->buf->buf.u_buffer
-                                         : ref_buf->buf->buf.v_buffer;
+        pd->pre[ref].buf0 =
+            (plane == 1) ? ref_buf->buf.u_buffer : ref_buf->buf.v_buffer;
         pd->pre[ref].buf =
-            pd->pre[ref].buf0 +
-            scaled_buffer_offset(pre_x, pre_y, ref_buf->buf->buf.uv_stride,
-                                 &ref_buf->sf);
-        pd->pre[ref].width = ref_buf->buf->buf.uv_crop_width;
-        pd->pre[ref].height = ref_buf->buf->buf.uv_crop_height;
-        pd->pre[ref].stride = ref_buf->buf->buf.uv_stride;
+            pd->pre[ref].buf0 + scaled_buffer_offset(pre_x, pre_y,
+                                                     ref_buf->buf.uv_stride,
+                                                     ref_scale_factors);
+        pd->pre[ref].width = ref_buf->buf.uv_crop_width;
+        pd->pre[ref].height = ref_buf->buf.uv_crop_height;
+        pd->pre[ref].stride = ref_buf->buf.uv_stride;
 
         const struct scale_factors *const sf =
-            is_intrabc ? &cm->sf_identity : &ref_buf->sf;
+            is_intrabc ? &cm->sf_identity : ref_scale_factors;
         struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref];
 
         const MV mv = this_mbmi->mv[ref].as_mv;
@@ -769,7 +770,7 @@
     int src_stride[2];
     for (ref = 0; ref < 1 + is_compound; ++ref) {
       const struct scale_factors *const sf =
-          is_intrabc ? &cm->sf_identity : &xd->block_refs[ref]->sf;
+          is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref];
       struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref];
       const MV mv = mi->mv[ref].as_mv;
       PadBlock block;
@@ -806,7 +807,7 @@
 
     for (ref = 0; ref < 1 + is_compound; ++ref) {
       const struct scale_factors *const sf =
-          is_intrabc ? &cm->sf_identity : &xd->block_refs[ref]->sf;
+          is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref];
       WarpTypesAllowed warp_types;
       warp_types.global_warp_allowed = is_global[ref];
       warp_types.local_warp_allowed = mi->motion_mode == WARPED_CAUSAL;
@@ -1063,11 +1064,13 @@
       assert(frame == INTRA_FRAME);
       assert(ref == 0);
     } else {
-      RefBuffer *ref_buf = &cm->current_frame.frame_refs[frame - LAST_FRAME];
+      const RefCntBuffer *ref_buf = get_ref_frame_buf(cm, frame);
+      const struct scale_factors *ref_scale_factors =
+          get_ref_scale_factors_const(cm, frame);
 
-      xd->block_refs[ref] = ref_buf;
-      av1_setup_pre_planes(xd, ref, &ref_buf->buf->buf, mi_row, mi_col,
-                           &ref_buf->sf, num_planes);
+      xd->block_ref_scale_factors[ref] = ref_scale_factors;
+      av1_setup_pre_planes(xd, ref, &ref_buf->buf, mi_row, mi_col,
+                           ref_scale_factors, num_planes);
     }
   }
 
@@ -2437,17 +2440,28 @@
   int width, height;
   int found = 0;
   int has_valid_ref_frame = 0;
-  for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
+  for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
     if (aom_rb_read_bit(rb)) {
-      YV12_BUFFER_CONFIG *const buf = &cm->current_frame.frame_refs[i].buf->buf;
-      width = buf->y_crop_width;
-      height = buf->y_crop_height;
-      cm->render_width = buf->render_width;
-      cm->render_height = buf->render_height;
-      setup_superres(cm, rb, &width, &height);
-      resize_context_buffers(cm, width, height);
-      found = 1;
-      break;
+      const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, i);
+      // This will never be NULL in a normal stream, as streams are required to
+      // have a shown keyframe before any inter frames, which would refresh all
+      // the reference buffers. However, it might be null if we're starting in
+      // the middle of a stream, and static analysis will error if we don't do
+      // a null check here.
+      if (ref_buf == NULL) {
+        aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                           "Invalid condition: invalid reference buffer");
+      } else {
+        const YV12_BUFFER_CONFIG *const buf = &ref_buf->buf;
+        width = buf->y_crop_width;
+        height = buf->y_crop_height;
+        cm->render_width = buf->render_width;
+        cm->render_height = buf->render_height;
+        setup_superres(cm, rb, &width, &height);
+        resize_context_buffers(cm, width, height);
+        found = 1;
+        break;
+      }
     }
   }
 
@@ -2468,20 +2482,20 @@
 
   // Check to make sure at least one of frames that this frame references
   // has valid dimensions.
-  for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    RefBuffer *const ref_frame = &cm->current_frame.frame_refs[i];
+  for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
+    const RefCntBuffer *const ref_frame = get_ref_frame_buf(cm, i);
     has_valid_ref_frame |=
-        valid_ref_frame_size(ref_frame->buf->buf.y_crop_width,
-                             ref_frame->buf->buf.y_crop_height, width, height);
+        valid_ref_frame_size(ref_frame->buf.y_crop_width,
+                             ref_frame->buf.y_crop_height, width, height);
   }
   if (!has_valid_ref_frame)
     aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                        "Referenced frame has invalid size");
-  for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    RefBuffer *const ref_frame = &cm->current_frame.frame_refs[i];
+  for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
+    const RefCntBuffer *const ref_frame = get_ref_frame_buf(cm, i);
     if (!valid_ref_frame_img_fmt(
-            ref_frame->buf->buf.bit_depth, ref_frame->buf->buf.subsampling_x,
-            ref_frame->buf->buf.subsampling_y, seq_params->bit_depth,
+            ref_frame->buf.bit_depth, ref_frame->buf.subsampling_x,
+            ref_frame->buf.subsampling_y, seq_params->bit_depth,
             seq_params->subsampling_x, seq_params->subsampling_y))
       aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                          "Referenced frame has incompatible color format");
@@ -4713,7 +4727,7 @@
   pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1;
 
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    cm->current_frame.frame_refs[i].buf = NULL;
+    cm->remapped_ref_idx[i] = INVALID_IDX;
   }
 
   if (pbi->need_resync) {
@@ -5007,7 +5021,7 @@
       pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1;
 
     for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-      cm->current_frame.frame_refs[i].buf = NULL;
+      cm->remapped_ref_idx[i] = INVALID_IDX;
     }
     if (pbi->need_resync) {
       reset_ref_frame_map(cm);
@@ -5128,7 +5142,6 @@
         int ref = 0;
         if (!frame_refs_short_signaling) {
           ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2);
-          RefCntBuffer *const buf = cm->ref_frame_map[ref];
 
           // Most of the time, streams start with a keyframe. In that case,
           // ref_frame_map will have been filled in at that point and will not
@@ -5136,15 +5149,12 @@
           // with an intra-only frame, so long as they don't then signal a
           // reference to a slot that hasn't been set yet. That's what we are
           // checking here.
-          if (buf == NULL)
+          if (cm->ref_frame_map[ref] == NULL)
             aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                                "Inter frame requests nonexistent reference");
-
-          RefBuffer *const ref_frame = &cm->current_frame.frame_refs[i];
-          ref_frame->buf = buf;
-          ref_frame->map_idx = ref;
+          cm->remapped_ref_idx[i] = ref;
         } else {
-          ref = cm->current_frame.frame_refs[i].map_idx;
+          ref = cm->remapped_ref_idx[i];
         }
 
         cm->ref_frame_sign_bias[LAST_FRAME + i] = 0;
@@ -5181,9 +5191,9 @@
       cm->switchable_motion_mode = aom_rb_read_bit(rb);
     }
 
-    cm->prev_frame = get_prev_frame(cm);
+    cm->prev_frame = get_primary_ref_frame_buf(cm);
     if (cm->primary_ref_frame != PRIMARY_REF_NONE &&
-        cm->current_frame.frame_refs[cm->primary_ref_frame].buf == NULL) {
+        get_primary_ref_frame_buf(cm) == NULL) {
       aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                          "Reference frame containing this frame's initial "
                          "frame context is unavailable.");
@@ -5196,12 +5206,14 @@
       else
         cm->allow_ref_frame_mvs = 0;
 
-      for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-        RefBuffer *const ref_buf = &cm->current_frame.frame_refs[i];
+      for (int i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
+        const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, i);
+        struct scale_factors *const ref_scale_factors =
+            get_ref_scale_factors(cm, i);
         av1_setup_scale_factors_for_frame(
-            &ref_buf->sf, ref_buf->buf->buf.y_crop_width,
-            ref_buf->buf->buf.y_crop_height, cm->width, cm->height);
-        if ((!av1_is_valid_scale(&ref_buf->sf)))
+            ref_scale_factors, ref_buf->buf.y_crop_width,
+            ref_buf->buf.y_crop_height, cm->width, cm->height);
+        if ((!av1_is_valid_scale(ref_scale_factors)))
           aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
                              "Reference frame has invalid dimensions");
       }
@@ -5480,8 +5492,7 @@
     // use the default frame context values
     *cm->fc = *cm->default_frame_context;
   } else {
-    *cm->fc =
-        cm->current_frame.frame_refs[cm->primary_ref_frame].buf->frame_context;
+    *cm->fc = get_primary_ref_frame_buf(cm)->frame_context;
   }
   if (!cm->fc->initialized)
     aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index febee86..fcdd5c7 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1388,9 +1388,7 @@
 
   for (int ref = 0; ref < 1 + has_second_ref(mbmi); ++ref) {
     const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref];
-    RefBuffer *ref_buf = &cm->current_frame.frame_refs[frame - LAST_FRAME];
-
-    xd->block_refs[ref] = ref_buf;
+    xd->block_ref_scale_factors[ref] = get_ref_scale_factors_const(cm, frame);
   }
 
   mbmi->motion_mode = SIMPLE_TRANSLATION;
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c
index 2aea93f..082e9ed 100644
--- a/av1/decoder/decoder.c
+++ b/av1/decoder/decoder.c
@@ -426,7 +426,7 @@
 
     // Invalidate these references until the next frame starts.
     for (ref_index = 0; ref_index < INTER_REFS_PER_FRAME; ref_index++) {
-      cm->current_frame.frame_refs[ref_index].buf = NULL;
+      cm->remapped_ref_idx[ref_index] = INVALID_IDX;
     }
   }
 }
@@ -448,9 +448,8 @@
     // TODO(jkoleszar): Error concealment is undefined and non-normative
     // at this point, but if it becomes so, [0] may not always be the correct
     // thing to do here.
-    if (cm->current_frame.frame_refs[0].buf != NULL) {
-      cm->current_frame.frame_refs[0].buf->buf.corrupted = 1;
-    }
+    RefCntBuffer *ref_buf = get_ref_frame_buf(cm, LAST_FRAME);
+    if (ref_buf != NULL) ref_buf->buf.corrupted = 1;
   }
 
   // Find a free buffer for the new frame, releasing the reference previously
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 58e9efe..b1f17da 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1934,13 +1934,9 @@
     aom_wb_write_bit(wb, lf->mode_ref_delta_update);
 
     if (lf->mode_ref_delta_update) {
-      const int prime_idx = cm->primary_ref_frame;
-      const RefCntBuffer *const buf =
-          prime_idx == PRIMARY_REF_NONE
-              ? NULL
-              : cm->current_frame.frame_refs[prime_idx].buf;
+      const RefCntBuffer *buf = get_primary_ref_frame_buf(cm);
       int8_t last_ref_deltas[REF_FRAMES];
-      if (prime_idx == PRIMARY_REF_NONE || buf == NULL) {
+      if (buf == NULL) {
         av1_set_default_ref_deltas(last_ref_deltas);
       } else {
         memcpy(last_ref_deltas, buf->ref_deltas, REF_FRAMES);
@@ -1953,7 +1949,7 @@
       }
 
       int8_t last_mode_deltas[MAX_MODE_LF_DELTAS];
-      if (prime_idx == PRIMARY_REF_NONE || buf == NULL) {
+      if (buf == NULL) {
         av1_set_default_mode_deltas(last_mode_deltas);
       } else {
         memcpy(last_mode_deltas, buf->mode_deltas, MAX_MODE_LF_DELTAS);
@@ -2197,9 +2193,10 @@
 }
 
 static int get_refresh_mask(AV1_COMP *cpi) {
-  if ((cpi->common.current_frame.frame_type == KEY_FRAME &&
-       cpi->common.show_frame) ||
-      frame_is_sframe(&cpi->common))
+  const AV1_COMMON *const cm = &cpi->common;
+
+  if ((cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) ||
+      frame_is_sframe(cm))
     return 0xFF;
 
   int refresh_mask = 0;
@@ -2213,7 +2210,7 @@
   //     shifted and become the new virtual indexes for LAST2_FRAME and
   //     LAST3_FRAME.
   refresh_mask |=
-      (cpi->refresh_last_frame << get_ref_frame_map_idx(cpi, LAST3_FRAME));
+      (cpi->refresh_last_frame << get_ref_frame_map_idx(cm, LAST3_FRAME));
 
 #if USE_SYMM_MULTI_LAYER
   const int bwd_ref_frame =
@@ -2222,10 +2219,10 @@
   const int bwd_ref_frame = BWDREF_FRAME;
 #endif
   refresh_mask |=
-      (cpi->refresh_bwd_ref_frame << get_ref_frame_map_idx(cpi, bwd_ref_frame));
+      (cpi->refresh_bwd_ref_frame << get_ref_frame_map_idx(cm, bwd_ref_frame));
 
-  refresh_mask |= (cpi->refresh_alt2_ref_frame
-                   << get_ref_frame_map_idx(cpi, ALTREF2_FRAME));
+  refresh_mask |=
+      (cpi->refresh_alt2_ref_frame << get_ref_frame_map_idx(cm, ALTREF2_FRAME));
 
   if (av1_preserve_existing_gf(cpi)) {
     // We have decided to preserve the previously existing golden frame as our
@@ -2243,13 +2240,13 @@
       return refresh_mask;
     } else {
       return refresh_mask | (cpi->refresh_golden_frame
-                             << get_ref_frame_map_idx(cpi, ALTREF_FRAME));
+                             << get_ref_frame_map_idx(cm, ALTREF_FRAME));
     }
   } else {
-    const int arf_idx = get_ref_frame_map_idx(cpi, ALTREF_FRAME);
+    const int arf_idx = get_ref_frame_map_idx(cm, ALTREF_FRAME);
     return refresh_mask |
            (cpi->refresh_golden_frame
-            << get_ref_frame_map_idx(cpi, GOLDEN_FRAME)) |
+            << get_ref_frame_map_idx(cm, GOLDEN_FRAME)) |
            (cpi->refresh_alt_ref_frame << arf_idx);
   }
 }
@@ -2354,14 +2351,13 @@
   write_render_size(cm, wb);
 }
 
-static void write_frame_size_with_refs(AV1_COMP *cpi,
+static void write_frame_size_with_refs(const AV1_COMMON *const cm,
                                        struct aom_write_bit_buffer *wb) {
-  AV1_COMMON *const cm = &cpi->common;
   int found = 0;
 
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, ref_frame);
+    const YV12_BUFFER_CONFIG *cfg = get_ref_frame_yv12_buf(cm, ref_frame);
 
     if (cfg != NULL) {
       found = cm->superres_upscaled_width == cfg->y_crop_width &&
@@ -2545,7 +2541,7 @@
   if (!pars->update_parameters) {
     int ref_frame, ref_idx;
     for (ref_frame = LAST_FRAME; ref_frame < REF_FRAMES; ref_frame++) {
-      ref_idx = get_ref_frame_map_idx(cpi, ref_frame);
+      ref_idx = get_ref_frame_map_idx(cm, ref_frame);
       assert(ref_idx != INVALID_IDX);
       const RefCntBuffer *const buf = cm->ref_frame_map[ref_idx];
       if (buf->film_grain_params_present &&
@@ -2804,7 +2800,7 @@
     // does not work currently and causes mismatches when resize is on.
     // Fix it before turning the optimization back on.
     /*
-    YV12_BUFFER_CONFIG *ref_buf = get_ref_frame_buffer(cpi, frame);
+    YV12_BUFFER_CONFIG *ref_buf = get_ref_frame_yv12_buf(cpi, frame);
     if (cpi->source->y_crop_width == ref_buf->y_crop_width &&
         cpi->source->y_crop_height == ref_buf->y_crop_height) {
       write_global_motion_params(&cm->global_motion[frame],
@@ -2825,14 +2821,12 @@
   }
 }
 
-static int check_frame_refs_short_signaling(AV1_COMP *const cpi) {
-  AV1_COMMON *const cm = &cpi->common;
-
+static int check_frame_refs_short_signaling(AV1_COMMON *const cm) {
   // Check whether all references are distinct frames.
   const RefCntBuffer *seen_bufs[FRAME_BUFFERS] = { NULL };
   int num_refs = 0;
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     if (buf != NULL) {
       int seen = 0;
       for (int i = 0; i < num_refs; i++) {
@@ -2855,14 +2849,17 @@
 
   // Check whether the encoder side ref frame choices are aligned with that to
   // be derived at the decoder side.
-  RefBuffer frame_refs_copy[INTER_REFS_PER_FRAME];
+  int remapped_ref_idx_copy[REF_FRAMES];
+  struct scale_factors ref_scale_factors_copy[REF_FRAMES];
 
   // Backup the frame refs info
-  memcpy(frame_refs_copy, cm->current_frame.frame_refs,
-         INTER_REFS_PER_FRAME * sizeof(RefBuffer));
+  memcpy(remapped_ref_idx_copy, cm->remapped_ref_idx,
+         REF_FRAMES * sizeof(*remapped_ref_idx_copy));
+  memcpy(ref_scale_factors_copy, cm->ref_scale_factors,
+         REF_FRAMES * sizeof(*ref_scale_factors_copy));
 
-  const int lst_map_idx = get_ref_frame_map_idx(cpi, LAST_FRAME);
-  const int gld_map_idx = get_ref_frame_map_idx(cpi, GOLDEN_FRAME);
+  const int lst_map_idx = get_ref_frame_map_idx(cm, LAST_FRAME);
+  const int gld_map_idx = get_ref_frame_map_idx(cm, GOLDEN_FRAME);
 
   // Set up the frame refs mapping indexes according to the
   // frame_refs_short_signaling policy.
@@ -2874,8 +2871,10 @@
   for (int ref_idx = 0; ref_idx < INTER_REFS_PER_FRAME; ++ref_idx) {
     // Compare the buffer index between two reference frames indexed
     // respectively by the encoder and the decoder side decisions.
-    if (cm->current_frame.frame_refs[ref_idx].buf !=
-        frame_refs_copy[ref_idx].buf) {
+    RefCntBuffer *ref_frame_buf_copy = NULL;
+    if (remapped_ref_idx_copy[ref_idx] != INVALID_IDX)
+      ref_frame_buf_copy = cm->ref_frame_map[remapped_ref_idx_copy[ref_idx]];
+    if (get_ref_frame_buf(cm, LAST_FRAME + ref_idx) != ref_frame_buf_copy) {
       frame_refs_short_signaling = 0;
       break;
     }
@@ -2887,16 +2886,18 @@
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     printf("enc_ref(map_idx=%d)=%d, vs. "
         "dec_ref(map_idx=%d)=%d\n",
-        get_ref_frame_map_idx(cpi, ref_frame), ref_frame,
-        cm->current_frame.frame_refs[ref_frame - LAST_FRAME].map_idx,
+        get_ref_frame_map_idx(cm, ref_frame), ref_frame,
+        cm->remapped_ref_idx[ref_frame - LAST_FRAME],
         ref_frame);
   }
 #endif  // 0
 
   // Restore the frame refs info if frame_refs_short_signaling is off.
   if (!frame_refs_short_signaling) {
-    memcpy(cm->current_frame.frame_refs, frame_refs_copy,
-           INTER_REFS_PER_FRAME * sizeof(RefBuffer));
+    memcpy(cm->remapped_ref_idx, remapped_ref_idx_copy,
+           REF_FRAMES * sizeof(*remapped_ref_idx_copy));
+    memcpy(cm->ref_scale_factors, ref_scale_factors_copy,
+           REF_FRAMES * sizeof(*ref_scale_factors_copy));
   }
   return frame_refs_short_signaling;
 }
@@ -3137,27 +3138,27 @@
         //   An example solution for encoder-side implementation on frame refs
         //   short signaling, which is only turned on when the encoder side
         //   decision on ref frames is identical to that at the decoder side.
-        frame_refs_short_signaling = check_frame_refs_short_signaling(cpi);
+        frame_refs_short_signaling = check_frame_refs_short_signaling(cm);
       }
 
       if (seq_params->order_hint_info.enable_order_hint)
         aom_wb_write_bit(wb, frame_refs_short_signaling);
 
       if (frame_refs_short_signaling) {
-        const int lst_ref = get_ref_frame_map_idx(cpi, LAST_FRAME);
+        const int lst_ref = get_ref_frame_map_idx(cm, LAST_FRAME);
         aom_wb_write_literal(wb, lst_ref, REF_FRAMES_LOG2);
 
-        const int gld_ref = get_ref_frame_map_idx(cpi, GOLDEN_FRAME);
+        const int gld_ref = get_ref_frame_map_idx(cm, GOLDEN_FRAME);
         aom_wb_write_literal(wb, gld_ref, REF_FRAMES_LOG2);
       }
 
       for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-        assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
+        assert(get_ref_frame_map_idx(cm, ref_frame) != INVALID_IDX);
         if (!frame_refs_short_signaling)
-          aom_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame),
+          aom_wb_write_literal(wb, get_ref_frame_map_idx(cm, ref_frame),
                                REF_FRAMES_LOG2);
         if (seq_params->frame_id_numbers_present_flag) {
-          int i = get_ref_frame_map_idx(cpi, ref_frame);
+          int i = get_ref_frame_map_idx(cm, ref_frame);
           int frame_id_len = seq_params->frame_id_length;
           int diff_len = seq_params->delta_frame_id_length;
           int delta_frame_id_minus_1 =
@@ -3175,7 +3176,7 @@
       }
 
       if (!cm->error_resilient_mode && frame_size_override_flag) {
-        write_frame_size_with_refs(cpi, wb);
+        write_frame_size_with_refs(cm, wb);
       } else {
         write_frame_size(cm, frame_size_override_flag, wb);
       }
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 8ce3119..e50f213 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -188,7 +188,8 @@
                                                    BLOCK_SIZE bs) {
   unsigned int sse, var;
   uint8_t *last_y;
-  const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME);
+  const YV12_BUFFER_CONFIG *last =
+      get_ref_frame_yv12_buf(&cpi->common, LAST_FRAME);
 
   assert(last != NULL);
   last_y =
@@ -2118,7 +2119,7 @@
   mbmi->ref_frame[1] = NONE_FRAME;
   mbmi->motion_mode = SIMPLE_TRANSLATION;
 
-  YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref);
+  const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_yv12_buf(cm, ref);
   const YV12_BUFFER_CONFIG *scaled_ref_frame =
       av1_get_scaled_ref_frame(cpi, ref);
   struct buf_2d backup_yv12;
@@ -2140,8 +2141,7 @@
                          num_planes);
   } else {
     av1_setup_pre_planes(xd, ref_idx, yv12, mi_row, mi_col,
-                         &cm->current_frame.frame_refs[ref - LAST_FRAME].sf,
-                         num_planes);
+                         get_ref_scale_factors(cm, ref), num_planes);
   }
 
   // This overwrites the mv_limits so we will need to restore it later.
@@ -5502,8 +5502,7 @@
       // Retain GOLDEN/ALTERF
       if (ref_frame == GOLDEN_FRAME || ref_frame == ALTREF_FRAME) continue;
 
-      const RefCntBuffer *const buf =
-          cm->current_frame.frame_refs[ref_frame - LAST_FRAME].buf;
+      const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
       if (buf != NULL) {
         const unsigned int ref_order_hint = buf->order_hint;
 
@@ -5571,8 +5570,8 @@
   assert(!frame_is_intra_only(cm));
 
   int one_sided_refs = 1;
-  for (int ref = 0; ref < INTER_REFS_PER_FRAME; ++ref) {
-    const RefCntBuffer *const buf = cm->current_frame.frame_refs[ref].buf;
+  for (int ref = LAST_FRAME; ref <= ALTREF_FRAME; ++ref) {
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref);
     if (buf == NULL) continue;
 
     const int ref_order_hint = buf->order_hint;
@@ -5592,9 +5591,9 @@
   if (!skip_mode_info->skip_mode_allowed) return;
 
   const RefCntBuffer *const buf_0 =
-      cm->current_frame.frame_refs[skip_mode_info->ref_frame_idx_0].buf;
+      get_ref_frame_buf(cm, LAST_FRAME + skip_mode_info->ref_frame_idx_0);
   const RefCntBuffer *const buf_1 =
-      cm->current_frame.frame_refs[skip_mode_info->ref_frame_idx_1].buf;
+      get_ref_frame_buf(cm, LAST_FRAME + skip_mode_info->ref_frame_idx_1);
   assert(buf_0 != NULL && buf_1 != NULL);
 
   ref_order_hint[0] = buf_0->order_hint;
@@ -5867,7 +5866,9 @@
     int num_refs_using_gm = 0;
 
     for (frame = ALTREF_FRAME; frame >= LAST_FRAME; --frame) {
-      ref_buf[frame] = get_ref_frame_buffer(cpi, frame);
+      ref_buf[frame] = NULL;
+      RefCntBuffer *buf = get_ref_frame_buf(cm, frame);
+      if (buf != NULL) ref_buf[frame] = &buf->buf;
       int pframe;
       cm->global_motion[frame] = default_warp_params;
       const WarpedMotionParams *ref_params =
@@ -6357,10 +6358,11 @@
 
     set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]);
     for (ref = 0; ref < 1 + is_compound; ++ref) {
-      YV12_BUFFER_CONFIG *cfg = get_ref_frame_buffer(cpi, mbmi->ref_frame[ref]);
+      const YV12_BUFFER_CONFIG *cfg =
+          get_ref_frame_yv12_buf(cm, mbmi->ref_frame[ref]);
       assert(IMPLIES(!is_intrabc_block(mbmi), cfg));
       av1_setup_pre_planes(xd, ref, cfg, mi_row, mi_col,
-                           &xd->block_refs[ref]->sf, num_planes);
+                           xd->block_ref_scale_factors[ref], num_planes);
     }
 
     av1_build_inter_predictors_sb(cm, xd, mi_row, mi_col, NULL, bsize);
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 16a9b1d..d655154 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -383,8 +383,8 @@
       cm->fb_of_context_type[i] = -1;
     }
     cm->fb_of_context_type[REGULAR_FRAME] =
-        cm->show_frame ? get_ref_frame_map_idx(cpi, GOLDEN_FRAME)
-                       : get_ref_frame_map_idx(cpi, ALTREF_FRAME);
+        cm->show_frame ? get_ref_frame_map_idx(cm, GOLDEN_FRAME)
+                       : get_ref_frame_map_idx(cm, ALTREF_FRAME);
     cm->frame_context_idx = REGULAR_FRAME;
   } else {
     const GF_GROUP *gf_group = &cpi->twopass.gf_group;
@@ -402,7 +402,7 @@
       cm->frame_context_idx = REGULAR_FRAME;
     int wanted_fb = cm->fb_of_context_type[cm->frame_context_idx];
     for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
-      int fb = get_ref_frame_map_idx(cpi, ref_frame);
+      int fb = get_ref_frame_map_idx(cm, ref_frame);
       if (fb == wanted_fb) {
         cm->primary_ref_frame = ref_frame - LAST_FRAME;
       }
@@ -420,19 +420,18 @@
     av1_zero(cpi->interp_filter_selected);
     set_sb_size(&cm->seq_params, select_sb_size(cpi));
   } else {
-    if (cm->primary_ref_frame == PRIMARY_REF_NONE ||
-        cm->current_frame.frame_refs[cm->primary_ref_frame].buf == NULL) {
+    const RefCntBuffer *const primary_ref_buf = get_primary_ref_frame_buf(cm);
+    if (primary_ref_buf == NULL) {
       av1_setup_past_independence(cm);
       cm->seg.update_map = 1;
       cm->seg.update_data = 1;
     } else {
-      *cm->fc = cm->current_frame.frame_refs[cm->primary_ref_frame]
-                    .buf->frame_context;
+      *cm->fc = primary_ref_buf->frame_context;
     }
     av1_zero(cpi->interp_filter_selected[0]);
   }
 
-  cm->prev_frame = get_prev_frame(cm);
+  cm->prev_frame = get_primary_ref_frame_buf(cm);
   cpi->vaq_refresh = 0;
 }
 
@@ -978,7 +977,7 @@
 static void init_buffer_indices(AV1_COMP *cpi) {
   int fb_idx;
   for (fb_idx = 0; fb_idx < REF_FRAMES; ++fb_idx)
-    cpi->remapped_ref_idx[fb_idx] = fb_idx;
+    cpi->common.remapped_ref_idx[fb_idx] = fb_idx;
   cpi->rate_index = 0;
   cpi->rate_size = 0;
 }
@@ -3295,7 +3294,7 @@
       //       the last_fb_idxes[0] after reference frame buffer update
       cpi->rc.is_last_bipred_frame = 0;
       cm->show_existing_frame = 1;
-      cpi->existing_fb_idx_to_show = cpi->remapped_ref_idx[0];
+      cpi->existing_fb_idx_to_show = cm->remapped_ref_idx[0];
 #if USE_SYMM_MULTI_LAYER
     }
 #endif
@@ -3314,8 +3313,8 @@
     cpi->rc.is_src_frame_alt_ref = 1;
     cpi->existing_fb_idx_to_show =
         (next_frame_update_type == OVERLAY_UPDATE)
-            ? get_ref_frame_map_idx(cpi, ALTREF_FRAME)
-            : get_ref_frame_map_idx(cpi, bwdref_to_show);
+            ? get_ref_frame_map_idx(cm, ALTREF_FRAME)
+            : get_ref_frame_map_idx(cm, bwdref_to_show);
 #if USE_SYMM_MULTI_LAYER
     if (cpi->new_bwdref_update_rule == 0)
 #endif
@@ -3479,7 +3478,7 @@
     char file_name[256] = "";
     snprintf(file_name, sizeof(file_name), "/tmp/enc_F%d_ref_%d.yuv",
              cm->current_frame.frame_number, ref_frame);
-    dump_one_image(cm, get_ref_frame_buffer(cpi, ref_frame), file_name);
+    dump_one_image(cm, get_ref_frame_yv12_buf(cpi, ref_frame), file_name);
   }
 }
 #endif  // DUMP_REF_FRAME_IMAGES == 1
@@ -3492,7 +3491,8 @@
   // TODO(isbs): shift the scaled indices as well
   for (int ref_frame = LAST3_FRAME; ref_frame > LAST_FRAME; --ref_frame) {
     const int ref_idx = ref_frame - LAST_FRAME;
-    cpi->remapped_ref_idx[ref_idx] = cpi->remapped_ref_idx[ref_idx - 1];
+    cpi->common.remapped_ref_idx[ref_idx] =
+        cpi->common.remapped_ref_idx[ref_idx - 1];
 
     if (!cpi->rc.is_src_frame_alt_ref) {
       memcpy(cpi->interp_filter_selected[ref_frame],
@@ -3518,8 +3518,8 @@
            cpi->interp_filter_selected[ordered_bwd[i - 1]],
            sizeof(cpi->interp_filter_selected[ordered_bwd[i - 1]]));
 
-    cpi->remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] =
-        cpi->remapped_ref_idx[ordered_bwd[i - 1] - LAST_FRAME];
+    cpi->common.remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] =
+        cpi->common.remapped_ref_idx[ordered_bwd[i - 1] - LAST_FRAME];
   }
 }
 
@@ -3538,8 +3538,8 @@
            cpi->interp_filter_selected[ordered_bwd[i + 1]],
            sizeof(cpi->interp_filter_selected[ordered_bwd[i + 1]]));
 
-    cpi->remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] =
-        cpi->remapped_ref_idx[ordered_bwd[i + 1] - LAST_FRAME];
+    cpi->common.remapped_ref_idx[ordered_bwd[i] - LAST_FRAME] =
+        cpi->common.remapped_ref_idx[ordered_bwd[i + 1] - LAST_FRAME];
   }
 }
 #endif  // USE_SYMM_MULTI_LAYER
@@ -3579,8 +3579,8 @@
   if ((cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) ||
       frame_is_sframe(cm)) {
     for (int ref_frame = 0; ref_frame < REF_FRAMES; ++ref_frame) {
-      assign_frame_buffer_p(
-          &cm->ref_frame_map[cpi->remapped_ref_idx[ref_frame]], cm->cur_frame);
+      assign_frame_buffer_p(&cm->ref_frame_map[cm->remapped_ref_idx[ref_frame]],
+                            cm->cur_frame);
     }
     return;
   }
@@ -3601,14 +3601,14 @@
 
     if (!cpi->preserve_arf_as_gld) {
       assign_frame_buffer_p(
-          &cm->ref_frame_map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)],
+          &cm->ref_frame_map[get_ref_frame_map_idx(cm, ALTREF_FRAME)],
           cm->cur_frame);
     }
 
-    tmp = get_ref_frame_map_idx(cpi, ALTREF_FRAME);
-    cpi->remapped_ref_idx[ALTREF_FRAME - 1] =
-        get_ref_frame_map_idx(cpi, GOLDEN_FRAME);
-    cpi->remapped_ref_idx[GOLDEN_FRAME - 1] = tmp;
+    tmp = get_ref_frame_map_idx(cm, ALTREF_FRAME);
+    cm->remapped_ref_idx[ALTREF_FRAME - LAST_FRAME] =
+        get_ref_frame_map_idx(cm, GOLDEN_FRAME);
+    cm->remapped_ref_idx[GOLDEN_FRAME - LAST_FRAME] = tmp;
 
     // TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to
     // cpi->interp_filter_selected[GOLDEN_FRAME]?
@@ -3626,11 +3626,11 @@
     // Deal with the special case for showing existing internal ALTREF_FRAME
     // Refresh the LAST_FRAME with the ALTREF_FRAME and retire the LAST3_FRAME
     // by updating the virtual indices.
-    const int last3_remapped_idx = get_ref_frame_map_idx(cpi, LAST3_FRAME);
+    const int last3_remapped_idx = get_ref_frame_map_idx(cm, LAST3_FRAME);
     shift_last_ref_frames(cpi);
 
-    cpi->remapped_ref_idx[LAST_FRAME - 1] =
-        get_ref_frame_map_idx(cpi, bwdref_to_show);
+    cm->remapped_ref_idx[LAST_FRAME - LAST_FRAME] =
+        get_ref_frame_map_idx(cm, bwdref_to_show);
 
     memcpy(cpi->interp_filter_selected[LAST_FRAME],
            cpi->interp_filter_selected[bwdref_to_show],
@@ -3640,17 +3640,17 @@
       lshift_bwd_ref_frames(cpi);
       // pass outdated forward reference frame (previous LAST3) to the
       // spared space
-      cpi->remapped_ref_idx[EXTREF_FRAME - 1] = last3_remapped_idx;
+      cm->remapped_ref_idx[EXTREF_FRAME - LAST_FRAME] = last3_remapped_idx;
     } else {
 #endif
-      cpi->remapped_ref_idx[bwdref_to_show - 1] = last3_remapped_idx;
+      cm->remapped_ref_idx[bwdref_to_show - LAST_FRAME] = last3_remapped_idx;
 #if USE_SYMM_MULTI_LAYER
     }
 #endif
   } else { /* For non key/golden frames */
     // === ALTREF_FRAME ===
     if (cpi->refresh_alt_ref_frame) {
-      int arf_idx = get_ref_frame_map_idx(cpi, ALTREF_FRAME);
+      int arf_idx = get_ref_frame_map_idx(cm, ALTREF_FRAME);
       assign_frame_buffer_p(&cm->ref_frame_map[arf_idx], cm->cur_frame);
 
       memcpy(cpi->interp_filter_selected[ALTREF_FRAME],
@@ -3661,7 +3661,7 @@
     // === GOLDEN_FRAME ===
     if (cpi->refresh_golden_frame) {
       assign_frame_buffer_p(
-          &cm->ref_frame_map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)],
+          &cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)],
           cm->cur_frame);
 
       memcpy(cpi->interp_filter_selected[GOLDEN_FRAME],
@@ -3677,15 +3677,15 @@
         // BWDREF -> ALTREF2 -> EXTREF
         // and assign the newly coded frame to BWDREF so that it always
         // keeps the nearest future frame
-        int tmp = get_ref_frame_map_idx(cpi, EXTREF_FRAME);
+        const int tmp = get_ref_frame_map_idx(cm, EXTREF_FRAME);
         assign_frame_buffer_p(&cm->ref_frame_map[tmp], cm->cur_frame);
 
         rshift_bwd_ref_frames(cpi);
-        cpi->remapped_ref_idx[BWDREF_FRAME - 1] = tmp;
+        cm->remapped_ref_idx[BWDREF_FRAME - LAST_FRAME] = tmp;
       } else {
 #endif  // USE_SYMM_MULTI_LAYER
         assign_frame_buffer_p(
-            &cm->ref_frame_map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)],
+            &cm->ref_frame_map[get_ref_frame_map_idx(cm, BWDREF_FRAME)],
             cm->cur_frame);
 #if USE_SYMM_MULTI_LAYER
       }
@@ -3698,7 +3698,7 @@
     // === ALTREF2_FRAME ===
     if (cpi->refresh_alt2_ref_frame) {
       assign_frame_buffer_p(
-          &cm->ref_frame_map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)],
+          &cm->ref_frame_map[get_ref_frame_map_idx(cm, ALTREF2_FRAME)],
           cm->cur_frame);
 
       memcpy(cpi->interp_filter_selected[ALTREF2_FRAME],
@@ -3739,13 +3739,13 @@
     //      v                      v                        v
     // remapped_ref_idx[2],   remapped_ref_idx[0],     remapped_ref_idx[1]
     assign_frame_buffer_p(
-        &cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST3_FRAME)],
+        &cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST3_FRAME)],
         cm->cur_frame);
 
-    int last3_remapped_idx = get_ref_frame_map_idx(cpi, LAST3_FRAME);
+    int last3_remapped_idx = get_ref_frame_map_idx(cm, LAST3_FRAME);
 
     shift_last_ref_frames(cpi);
-    cpi->remapped_ref_idx[LAST_FRAME - 1] = last3_remapped_idx;
+    cm->remapped_ref_idx[LAST_FRAME - LAST_FRAME] = last3_remapped_idx;
 
     assert(!encode_show_existing_frame(cm));
     memcpy(cpi->interp_filter_selected[LAST_FRAME],
@@ -3768,12 +3768,12 @@
       //       virtual index reshuffling for BWDREF, the encoder always
       //       specifies a LAST_BIPRED right before BWDREF and completes the
       //       reshuffling job accordingly.
-      last3_remapped_idx = get_ref_frame_map_idx(cpi, LAST3_FRAME);
+      last3_remapped_idx = get_ref_frame_map_idx(cm, LAST3_FRAME);
 
       shift_last_ref_frames(cpi);
-      cpi->remapped_ref_idx[LAST_FRAME - 1] =
-          get_ref_frame_map_idx(cpi, BWDREF_FRAME);
-      cpi->remapped_ref_idx[BWDREF_FRAME - 1] = last3_remapped_idx;
+      cm->remapped_ref_idx[LAST_FRAME - LAST_FRAME] =
+          get_ref_frame_map_idx(cm, BWDREF_FRAME);
+      cm->remapped_ref_idx[BWDREF_FRAME - LAST_FRAME] = last3_remapped_idx;
 
       memcpy(cpi->interp_filter_selected[LAST_FRAME],
              cpi->interp_filter_selected[BWDREF_FRAME],
@@ -3808,7 +3808,7 @@
     if (cpi->ref_frame_flags & ref_mask[ref_frame - 1]) {
       BufferPool *const pool = cm->buffer_pool;
       const YV12_BUFFER_CONFIG *const ref =
-          get_ref_frame_buffer(cpi, ref_frame);
+          get_ref_frame_yv12_buf(cm, ref_frame);
 
       if (ref == NULL) {
         cpi->scaled_ref_buf[ref_frame - 1] = NULL;
@@ -3848,7 +3848,7 @@
           alloc_frame_mvs(cm, new_fb);
         }
       } else {
-        RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+        RefCntBuffer *buf = get_ref_frame_buf(cm, ref_frame);
         buf->buf.y_crop_width = ref->y_crop_width;
         buf->buf.y_crop_height = ref->y_crop_height;
         cpi->scaled_ref_buf[ref_frame - 1] = buf;
@@ -4100,18 +4100,13 @@
   init_motion_estimation(cpi);
 
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    RefBuffer *const ref_buf =
-        &cm->current_frame.frame_refs[ref_frame - LAST_FRAME];
-    RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+    RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     if (buf != NULL) {
-      ref_buf->buf = buf;
-      av1_setup_scale_factors_for_frame(&ref_buf->sf, buf->buf.y_crop_width,
+      struct scale_factors *sf = get_ref_scale_factors(cm, ref_frame);
+      av1_setup_scale_factors_for_frame(sf, buf->buf.y_crop_width,
                                         buf->buf.y_crop_height, cm->width,
                                         cm->height);
-      if (av1_is_scaled(&ref_buf->sf))
-        aom_extend_frame_borders(&buf->buf, num_planes);
-    } else {
-      ref_buf->buf = NULL;
+      if (av1_is_scaled(sf)) aom_extend_frame_borders(&buf->buf, num_planes);
     }
   }
 
@@ -4583,8 +4578,7 @@
 
     // Base q-index may have changed, so we need to assign proper default coef
     // probs before every iteration.
-    if (cm->primary_ref_frame == PRIMARY_REF_NONE ||
-        cm->current_frame.frame_refs[cm->primary_ref_frame].buf == NULL) {
+    if (get_primary_ref_frame_buf(cm) == NULL) {
       av1_default_coef_probs(cm);
       av1_setup_frame_contexts(cm);
     }
@@ -4783,61 +4777,48 @@
 }
 
 static int get_ref_frame_flags(const AV1_COMP *cpi) {
-  const RefCntBuffer **map = (const RefCntBuffer **)cpi->common.ref_frame_map;
+  const AV1_COMMON *const cm = &cpi->common;
+  const RefCntBuffer *last_buf = get_ref_frame_buf(cm, LAST_FRAME);
+  const RefCntBuffer *last2_buf = get_ref_frame_buf(cm, LAST2_FRAME);
+  const RefCntBuffer *last3_buf = get_ref_frame_buf(cm, LAST3_FRAME);
+  const RefCntBuffer *golden_buf = get_ref_frame_buf(cm, GOLDEN_FRAME);
+  const RefCntBuffer *bwd_buf = get_ref_frame_buf(cm, BWDREF_FRAME);
+  const RefCntBuffer *alt2_buf = get_ref_frame_buf(cm, ALTREF2_FRAME);
+  const RefCntBuffer *alt_buf = get_ref_frame_buf(cm, ALTREF_FRAME);
 
   // No.1 Priority: LAST_FRAME
-  const int last2_is_last =
-      map[cpi->remapped_ref_idx[1]] == map[cpi->remapped_ref_idx[0]];
-  const int last3_is_last =
-      map[cpi->remapped_ref_idx[2]] == map[cpi->remapped_ref_idx[0]];
-  const int gld_is_last = map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)] ==
-                          map[cpi->remapped_ref_idx[0]];
-  const int bwd_is_last = map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)] ==
-                          map[cpi->remapped_ref_idx[0]];
-  const int alt2_is_last = map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)] ==
-                           map[cpi->remapped_ref_idx[0]];
-  const int alt_is_last = map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)] ==
-                          map[cpi->remapped_ref_idx[0]];
 
+  const int last2_is_last = (last2_buf == last_buf);
+  const int last3_is_last = (last3_buf == last_buf);
+  const int gld_is_last = (golden_buf == last_buf);
+  const int bwd_is_last = (bwd_buf == last_buf);
+  const int alt2_is_last = (alt2_buf == last_buf);
+  const int alt_is_last = (alt_buf == last_buf);
   // No.2 Priority: ALTREF_FRAME
-  const int last2_is_alt = map[cpi->remapped_ref_idx[1]] ==
-                           map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)];
-  const int last3_is_alt = map[cpi->remapped_ref_idx[2]] ==
-                           map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)];
-  const int gld_is_alt = map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)] ==
-                         map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)];
-  const int bwd_is_alt = map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)] ==
-                         map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)];
-  const int alt2_is_alt = map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)] ==
-                          map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)];
+
+  const int last2_is_alt = (last2_buf == alt_buf);
+  const int last3_is_alt = (last3_buf == alt_buf);
+  const int gld_is_alt = (golden_buf == alt_buf);
+  const int bwd_is_alt = (bwd_buf == alt_buf);
+  const int alt2_is_alt = (alt2_buf == alt_buf);
 
   // No.3 Priority: LAST2_FRAME
-  const int last3_is_last2 =
-      map[cpi->remapped_ref_idx[2]] == map[cpi->remapped_ref_idx[1]];
-  const int gld_is_last2 = map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)] ==
-                           map[cpi->remapped_ref_idx[1]];
-  const int bwd_is_last2 = map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)] ==
-                           map[cpi->remapped_ref_idx[1]];
-  const int alt2_is_last2 = map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)] ==
-                            map[cpi->remapped_ref_idx[1]];
+  const int last3_is_last2 = (last3_buf == last2_buf);
+  const int gld_is_last2 = (golden_buf == last2_buf);
+  const int bwd_is_last2 = (bwd_buf == last2_buf);
+  const int alt2_is_last2 = (alt2_buf == last2_buf);
 
   // No.4 Priority: LAST3_FRAME
-  const int gld_is_last3 = map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)] ==
-                           map[cpi->remapped_ref_idx[2]];
-  const int bwd_is_last3 = map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)] ==
-                           map[cpi->remapped_ref_idx[2]];
-  const int alt2_is_last3 = map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)] ==
-                            map[cpi->remapped_ref_idx[2]];
+  const int gld_is_last3 = (golden_buf == last3_buf);
+  const int bwd_is_last3 = (bwd_buf == last3_buf);
+  const int alt2_is_last3 = (alt2_buf == last3_buf);
 
   // No.5 Priority: GOLDEN_FRAME
-  const int bwd_is_gld = map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)] ==
-                         map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)];
-  const int alt2_is_gld = map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)] ==
-                          map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)];
+  const int bwd_is_gld = (bwd_buf == golden_buf);
+  const int alt2_is_gld = (alt2_buf == golden_buf);
 
   // No.6 Priority: BWDREF_FRAME
-  const int alt2_is_bwd = map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)] ==
-                          map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)];
+  const int alt2_is_bwd = (alt2_buf == bwd_buf);
 
   // No.7 Priority: ALTREF2_FRAME
 
@@ -4927,12 +4908,12 @@
       current_frame->frame_number, current_frame->order_hint, cm->show_frame,
       cm->show_existing_frame);
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    RefBuffer *buf = &cm->current_frame.frame_refs[ref_frame - LAST_FRAME];
-    const int ref_offset = (buf->buf) ? (int)buf->buf->order_hint : -1;
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
+    const int ref_offset = buf != NULL ? (int)buf->order_hint : -1;
     printf(" %d(%c-%d-%4.2f)", ref_offset,
            (cpi->ref_frame_flags & flag_list[ref_frame]) ? 'Y' : 'N',
-           (buf->buf) ? (int)buf->buf->frame_rf_level : -1,
-           (buf->buf) ? rate_factor_deltas[buf->buf->frame_rf_level] : -1);
+           buf ? (int)buf->frame_rf_level : -1,
+           buf ? rate_factor_deltas[buf->frame_rf_level] : -1);
   }
   printf(" ]\n");
 
@@ -4972,7 +4953,7 @@
   int ref_frame;
   printf("get_ref_frame_map_idx: [");
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame)
-    printf(" %d", get_ref_frame_map_idx(cpi, ref_frame));
+    printf(" %d", get_ref_frame_map_idx(cm, ref_frame));
   printf(" ]\n");
 #endif  // 0
 
@@ -5967,7 +5948,9 @@
   *tpl_group_frames = 0;
 
   // Initialize Golden reference frame.
-  gf_picture[0].frame = get_ref_frame_buffer(cpi, GOLDEN_FRAME);
+  gf_picture[0].frame = NULL;
+  RefCntBuffer *ref_buf = get_ref_frame_buf(cm, GOLDEN_FRAME);
+  if (ref_buf) gf_picture[0].frame = &ref_buf->buf;
   for (i = 0; i < 7; ++i) gf_picture[0].ref_frame[i] = -1;
   gld_index = 0;
   ++*tpl_group_frames;
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index a1daee4..6945cd3 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -618,26 +618,6 @@
   unsigned int row_mt;
   RefCntBuffer *scaled_ref_buf[INTER_REFS_PER_FRAME];
 
-  // For encoder, we have a two-level mapping from reference frame type to the
-  // corresponding buffer in the buffer pool:
-  // * 'remapped_ref_idx[i - 1]' maps reference type ‘i’ (range: LAST_FRAME ...
-  // EXTREF_FRAME) to a remapped index ‘j’ (in range: 0 ... REF_FRAMES - 1)
-  // * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to a pointer to
-  // the reference counted buffer structure RefCntBuffer, taken from the buffer
-  // pool cm->buffer_pool->frame_bufs.
-  //
-  // LAST_FRAME,                        ...,      EXTREF_FRAME
-  //      |                                           |
-  //      v                                           v
-  // remapped_ref_idx[LAST_FRAME - 1],  ...,  remapped_ref_idx[EXTREF_FRAME - 1]
-  //      |                                           |
-  //      v                                           v
-  // ref_frame_map[],                   ...,     ref_frame_map[]
-  //
-  // Note: INTRA_FRAME always refers to the current frame, so there's no need to
-  // have a remapped index for the same.
-  int remapped_ref_idx[REF_FRAMES];
-
   RefCntBuffer *last_show_frame_buf;  // last show frame buffer
 
   // refresh_*_frame are boolean flags. If 'refresh_xyz_frame' is true, then
@@ -915,41 +895,30 @@
          (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref);
 }
 
-static INLINE int get_ref_frame_map_idx(const AV1_COMP *cpi,
-                                        MV_REFERENCE_FRAME ref_frame) {
-  return (ref_frame >= LAST_FRAME)
-             ? cpi->remapped_ref_idx[ref_frame - LAST_FRAME]
-             : INVALID_IDX;
-}
-
-static INLINE RefCntBuffer *get_ref_frame_buf(const AV1_COMP *cpi,
-                                              MV_REFERENCE_FRAME ref_frame) {
-  const AV1_COMMON *const cm = &cpi->common;
-  const int map_idx = get_ref_frame_map_idx(cpi, ref_frame);
-  return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL;
-}
-
 // TODO(huisu@google.com, youzhou@microsoft.com): enable hash-me for HBD.
 static INLINE int av1_use_hash_me(const AV1_COMMON *const cm) {
   return cm->allow_screen_content_tools;
 }
 
 static INLINE hash_table *av1_get_ref_frame_hash_map(
-    const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) {
-  RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
-  return buf != NULL ? &buf->hash_table : NULL;
+    const AV1_COMMON *cm, MV_REFERENCE_FRAME ref_frame) {
+  const int map_idx = get_ref_frame_map_idx(cm, ref_frame);
+  RefCntBuffer *buf =
+      (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL;
+  return buf ? &buf->hash_table : NULL;
 }
 
-static INLINE YV12_BUFFER_CONFIG *get_ref_frame_buffer(
-    const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) {
-  RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+static INLINE const YV12_BUFFER_CONFIG *get_ref_frame_yv12_buf(
+    const AV1_COMMON *const cm, MV_REFERENCE_FRAME ref_frame) {
+  const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
   return buf != NULL ? &buf->buf : NULL;
 }
 
-static INLINE int enc_is_ref_frame_buf(AV1_COMP *cpi, RefCntBuffer *frame_buf) {
+static INLINE int enc_is_ref_frame_buf(const AV1_COMMON *const cm,
+                                       const RefCntBuffer *const frame_buf) {
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+    const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     if (buf == NULL) continue;
     if (frame_buf == buf) break;
   }
@@ -1011,10 +980,10 @@
 static INLINE void set_ref_ptrs(const AV1_COMMON *cm, MACROBLOCKD *xd,
                                 MV_REFERENCE_FRAME ref0,
                                 MV_REFERENCE_FRAME ref1) {
-  xd->block_refs[0] =
-      &cm->current_frame.frame_refs[ref0 >= LAST_FRAME ? ref0 - LAST_FRAME : 0];
-  xd->block_refs[1] =
-      &cm->current_frame.frame_refs[ref1 >= LAST_FRAME ? ref1 - LAST_FRAME : 0];
+  xd->block_ref_scale_factors[0] =
+      get_ref_scale_factors_const(cm, ref0 >= LAST_FRAME ? ref0 : 1);
+  xd->block_ref_scale_factors[1] =
+      get_ref_scale_factors_const(cm, ref1 >= LAST_FRAME ? ref1 : 1);
 }
 
 static INLINE int get_chessboard_index(int frame_index) {
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index 15cde17..292cf72 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -523,8 +523,9 @@
   TWO_PASS *twopass = &cpi->twopass;
   int recon_y_stride, recon_uv_stride, uv_mb_height;
 
-  YV12_BUFFER_CONFIG *const lst_yv12 = get_ref_frame_buffer(cpi, LAST_FRAME);
-  YV12_BUFFER_CONFIG *gld_yv12 = get_ref_frame_buffer(cpi, GOLDEN_FRAME);
+  const YV12_BUFFER_CONFIG *const lst_yv12 =
+      get_ref_frame_yv12_buf(cm, LAST_FRAME);
+  const YV12_BUFFER_CONFIG *gld_yv12 = get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
   YV12_BUFFER_CONFIG *const new_yv12 = &cm->cur_frame->buf;
   const YV12_BUFFER_CONFIG *first_ref_buf = lst_yv12;
   double intra_factor;
@@ -1061,8 +1062,8 @@
          DOUBLE_DIVIDE_CHECK(twopass->this_frame_stats.coded_error)) > 2.0))) {
     if (gld_yv12 != NULL) {
       assign_frame_buffer_p(
-          &cm->ref_frame_map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)],
-          cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST_FRAME)]);
+          &cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)],
+          cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)]);
     }
     twopass->sr_update_lag = 1;
   } else {
@@ -1073,16 +1074,15 @@
 
   // The frame we just compressed now becomes the last frame.
   assign_frame_buffer_p(
-      &cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST_FRAME)],
-      cm->cur_frame);
+      &cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)], cm->cur_frame);
 
   // Special case for the first frame. Copy into the GF buffer as a second
   // reference.
   if (current_frame->frame_number == 0 &&
-      get_ref_frame_map_idx(cpi, GOLDEN_FRAME) != INVALID_IDX) {
+      get_ref_frame_map_idx(cm, GOLDEN_FRAME) != INVALID_IDX) {
     assign_frame_buffer_p(
-        &cm->ref_frame_map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)],
-        cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST_FRAME)]);
+        &cm->ref_frame_map[get_ref_frame_map_idx(cm, GOLDEN_FRAME)],
+        cm->ref_frame_map[get_ref_frame_map_idx(cm, LAST_FRAME)]);
   }
 
   // Use this to see what the first pass reconstruction looks like.
diff --git a/av1/encoder/hash_motion.c b/av1/encoder/hash_motion.c
index e85a516..4c872f1 100644
--- a/av1/encoder/hash_motion.c
+++ b/av1/encoder/hash_motion.c
@@ -147,7 +147,8 @@
   }
 }
 
-int32_t av1_hash_table_count(hash_table *p_hash_table, uint32_t hash_value) {
+int32_t av1_hash_table_count(const hash_table *p_hash_table,
+                             uint32_t hash_value) {
   if (p_hash_table->p_lookup_table[hash_value] == NULL) {
     return 0;
   } else {
diff --git a/av1/encoder/hash_motion.h b/av1/encoder/hash_motion.h
index df3ec32..ed9bb6e 100644
--- a/av1/encoder/hash_motion.h
+++ b/av1/encoder/hash_motion.h
@@ -37,7 +37,8 @@
 void av1_hash_table_init(hash_table *p_hash_table, struct macroblock *x);
 void av1_hash_table_destroy(hash_table *p_hash_table);
 void av1_hash_table_create(hash_table *p_hash_table);
-int32_t av1_hash_table_count(hash_table *p_hash_table, uint32_t hash_value);
+int32_t av1_hash_table_count(const hash_table *p_hash_table,
+                             uint32_t hash_value);
 Iterator av1_hash_get_first_iterator(hash_table *p_hash_table,
                                      uint32_t hash_value);
 int32_t av1_has_exact_match(hash_table *p_hash_table, uint32_t hash_value1,
diff --git a/av1/encoder/mbgraph.c b/av1/encoder/mbgraph.c
index cc50458..4e37d32 100644
--- a/av1/encoder/mbgraph.c
+++ b/av1/encoder/mbgraph.c
@@ -364,7 +364,7 @@
 void av1_update_mbgraph_stats(AV1_COMP *cpi) {
   AV1_COMMON *const cm = &cpi->common;
   int i, n_frames = av1_lookahead_depth(cpi->lookahead);
-  YV12_BUFFER_CONFIG *golden_ref = get_ref_frame_buffer(cpi, GOLDEN_FRAME);
+  YV12_BUFFER_CONFIG *golden_ref = &get_ref_frame_buf(cm, GOLDEN_FRAME)->buf;
 
   assert(golden_ref != NULL);
 
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index 63b4947..482c2ee 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -2209,9 +2209,9 @@
 
         // for the hashMap
         hash_table *ref_frame_hash =
-            intra
-                ? &cpi->common.cur_frame->hash_table
-                : av1_get_ref_frame_hash_map(cpi, x->e_mbd.mi[0]->ref_frame[0]);
+            intra ? &cpi->common.cur_frame->hash_table
+                  : av1_get_ref_frame_hash_map(&cpi->common,
+                                               x->e_mbd.mi[0]->ref_frame[0]);
 
         av1_get_block_hash_value(
             what, what_stride, block_width, &hash_value1, &hash_value2,
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 1629a6a..c030d91 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -1259,7 +1259,8 @@
                                              int ref_frame) {
   assert(ref_frame >= LAST_FRAME && ref_frame <= ALTREF_FRAME);
   RefCntBuffer *const scaled_buf = cpi->scaled_ref_buf[ref_frame - 1];
-  RefCntBuffer *const ref_buf = get_ref_frame_buf(cpi, ref_frame);
+  const RefCntBuffer *const ref_buf =
+      get_ref_frame_buf(&cpi->common, ref_frame);
   return (scaled_buf != ref_buf && scaled_buf != NULL) ? &scaled_buf->buf
                                                        : NULL;
 }
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 4b4d433..696b179 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -7036,11 +7036,11 @@
     struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE]) {
   const AV1_COMMON *cm = &cpi->common;
   const int num_planes = av1_num_planes(cm);
-  const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame);
+  const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_yv12_buf(cm, ref_frame);
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = xd->mi[0];
   const struct scale_factors *const sf =
-      &cm->current_frame.frame_refs[ref_frame - 1].sf;
+      get_ref_scale_factors_const(cm, ref_frame);
   MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
 
   assert(yv12 != NULL);
@@ -8493,9 +8493,8 @@
   const int is_compound = has_second_ref(mbmi);
   assert(is_intrabc_block(mbmi) == 0);
   for (int j = 0; j < 1 + is_compound; ++j) {
-    const RefBuffer *ref_buf =
-        &cm->current_frame.frame_refs[mbmi->ref_frame[j] - LAST_FRAME];
-    const struct scale_factors *const sf = &ref_buf->sf;
+    const struct scale_factors *const sf =
+        get_ref_scale_factors_const(cm, mbmi->ref_frame[j]);
     // TODO(any): Refine skip flag calculation considering scaling
     if (av1_is_scaled(sf)) {
       skip_hor = 0;
@@ -10681,7 +10680,7 @@
           if (skip) continue;
         }
       }
-      assert(get_ref_frame_buffer(cpi, ref_frame) != NULL);
+      assert(get_ref_frame_yv12_buf(cm, ref_frame) != NULL);
       setup_buffer_ref_mvs_inter(cpi, x, ref_frame, bsize, mi_row, mi_col,
                                  yv12_mb);
     }
@@ -11179,8 +11178,7 @@
   if ((sf->selective_ref_frame >= 2) && comp_pred && !cpi->all_one_sided_refs) {
     unsigned int ref_offsets[2];
     for (int i = 0; i < 2; ++i) {
-      const RefCntBuffer *const buf =
-          cm->current_frame.frame_refs[ref_frame[i] - LAST_FRAME].buf;
+      const RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame[i]);
       assert(buf != NULL);
       ref_offsets[i] = buf->order_hint;
     }
diff --git a/av1/encoder/reconinter_enc.c b/av1/encoder/reconinter_enc.c
index 1100222..b19391e 100644
--- a/av1/encoder/reconinter_enc.c
+++ b/av1/encoder/reconinter_enc.c
@@ -143,22 +143,23 @@
         uint8_t *dst = dst_buf->buf + dst_buf->stride * y + x;
 
         ref = 0;
-        const RefBuffer *ref_buf =
-            &cm->current_frame
-                 .frame_refs[this_mbmi->ref_frame[ref] - LAST_FRAME];
+        const RefCntBuffer *ref_buf =
+            get_ref_frame_buf(cm, this_mbmi->ref_frame[ref]);
+        const struct scale_factors *ref_scale_factors =
+            get_ref_scale_factors_const(cm, this_mbmi->ref_frame[ref]);
 
-        pd->pre[ref].buf0 = (plane == 1) ? ref_buf->buf->buf.u_buffer
-                                         : ref_buf->buf->buf.v_buffer;
+        pd->pre[ref].buf0 =
+            (plane == 1) ? ref_buf->buf.u_buffer : ref_buf->buf.v_buffer;
         pd->pre[ref].buf =
-            pd->pre[ref].buf0 +
-            scaled_buffer_offset(pre_x, pre_y, ref_buf->buf->buf.uv_stride,
-                                 &ref_buf->sf);
-        pd->pre[ref].width = ref_buf->buf->buf.uv_crop_width;
-        pd->pre[ref].height = ref_buf->buf->buf.uv_crop_height;
-        pd->pre[ref].stride = ref_buf->buf->buf.uv_stride;
+            pd->pre[ref].buf0 + scaled_buffer_offset(pre_x, pre_y,
+                                                     ref_buf->buf.uv_stride,
+                                                     ref_scale_factors);
+        pd->pre[ref].width = ref_buf->buf.uv_crop_width;
+        pd->pre[ref].height = ref_buf->buf.uv_crop_height;
+        pd->pre[ref].stride = ref_buf->buf.uv_stride;
 
         const struct scale_factors *const sf =
-            is_intrabc ? &cm->sf_identity : &ref_buf->sf;
+            is_intrabc ? &cm->sf_identity : ref_scale_factors;
         struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref];
 
         const MV mv = this_mbmi->mv[ref].as_mv;
@@ -203,7 +204,7 @@
     uint8_t *const dst = dst_buf->buf;
     for (ref = 0; ref < 1 + is_compound; ++ref) {
       const struct scale_factors *const sf =
-          is_intrabc ? &cm->sf_identity : &xd->block_refs[ref]->sf;
+          is_intrabc ? &cm->sf_identity : xd->block_ref_scale_factors[ref];
       struct buf_2d *const pre_buf = is_intrabc ? dst_buf : &pd->pre[ref];
       const MV mv = mi->mv[ref].as_mv;
 
@@ -493,7 +494,7 @@
   struct macroblockd_plane *const pd = &xd->plane[plane];
   const MB_MODE_INFO *mi = xd->mi[0];
 
-  const struct scale_factors *const sf = &xd->block_refs[ref]->sf;
+  const struct scale_factors *const sf = xd->block_ref_scale_factors[ref];
   struct buf_2d *const pre_buf = &pd->pre[ref];
   uint8_t *const dst = get_buf_by_bd(xd, ext_dst) + ext_dst_stride * y + x;
   const MV mv = mi->mv[ref].as_mv;
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index d1d5ae1..d58537e 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -579,7 +579,8 @@
 static void temporal_filter_iterate_c(AV1_COMP *cpi,
                                       YV12_BUFFER_CONFIG **frames,
                                       int frame_count, int alt_ref_index,
-                                      int strength, RefBuffer *ref_buf) {
+                                      int strength,
+                                      struct scale_factors *ref_scale_factors) {
   const AV1_COMMON *cm = &cpi->common;
   const int num_planes = av1_num_planes(cm);
   int byte;
@@ -610,8 +611,8 @@
     predictor = predictor8;
   }
 
-  mbd->block_refs[0] = ref_buf;
-  mbd->block_refs[1] = ref_buf;
+  mbd->block_ref_scale_factors[0] = ref_scale_factors;
+  mbd->block_ref_scale_factors[1] = ref_scale_factors;
 
   for (i = 0; i < num_planes; i++) input_buffer[i] = mbd->plane[i].pre[0].buf;
 
@@ -677,8 +678,8 @@
               frames[frame]->u_buffer + mb_uv_offset,
               frames[frame]->v_buffer + mb_uv_offset, frames[frame]->y_stride,
               mb_uv_width, mb_uv_height, mbd->mi[0]->mv[0].as_mv.row,
-              mbd->mi[0]->mv[0].as_mv.col, predictor, &ref_buf->sf, mb_col * BW,
-              mb_row * BH, cm->allow_warped_motion, num_planes);
+              mbd->mi[0]->mv[0].as_mv.col, predictor, ref_scale_factors,
+              mb_col * BW, mb_row * BH, cm->allow_warped_motion, num_planes);
 
           // Apply the filter (YUV)
           if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
@@ -974,8 +975,7 @@
   int strength;
   int frames_to_blur_backward;
   int frames_to_blur_forward;
-  RefBuffer ref_buf;
-  ref_buf.buf = NULL;
+  struct scale_factors sf;
 
   YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS] = { NULL };
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
@@ -1020,7 +1020,7 @@
     // supported.
     // ARF is produced at the native frame size and resized when coded.
     av1_setup_scale_factors_for_frame(
-        &ref_buf.sf, frames[0]->y_crop_width, frames[0]->y_crop_height,
+        &sf, frames[0]->y_crop_width, frames[0]->y_crop_height,
         frames[0]->y_crop_width, frames[0]->y_crop_height);
   }
 
@@ -1031,5 +1031,5 @@
   av1_initialize_cost_tables(&cpi->common, &cpi->td.mb);
 
   temporal_filter_iterate_c(cpi, frames, frames_to_blur,
-                            frames_to_blur_backward, strength, &ref_buf);
+                            frames_to_blur_backward, strength, &sf);
 }