Remove idx from RefBuffer, use ptr instead

Instead of a RefBuffer storing its index in the array of buffers, store
a pointer to its RefCntBuffer.  This avoids storing redundant
information and in future can be replaced with a smart pointer for added
safety.

A number of attributes related to reference buffers are currently stored
as arrays with indices corresponding to the reference buffer array.
This commit also moves all of these to inside of the RefCntBuffer.

Also, rename RefCntBuffer.cur_frame_offset to order_hint.

This forms part of wider restructuring and refactoring in order to
achieve a clean API separation at the entry to the low-level encoder.

BUG=aomedia:2244

Change-Id: I10439f6d02c945433c6cc3d29e832f233be7b8be
diff --git a/av1/common/alloccommon.c b/av1/common/alloccommon.c
index c2a87a2..39b6b73 100644
--- a/av1/common/alloccommon.c
+++ b/av1/common/alloccommon.c
@@ -295,8 +295,8 @@
 
   aom_free(cm->fc);
   cm->fc = NULL;
-  aom_free(cm->frame_contexts);
-  cm->frame_contexts = NULL;
+  aom_free(cm->default_frame_context);
+  cm->default_frame_context = NULL;
 }
 
 void av1_init_context_buffers(AV1_COMMON *cm) { cm->setup_mi(cm); }
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 8555ef0..d6727b8 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -431,10 +431,11 @@
 #define BLOCK_OFFSET(x, i) \
   ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
 
+struct RefCntBuffer;
+
 typedef struct RefBuffer {
-  int idx;      // Index into 'cm->buffer_pool->frame_bufs'.
   int map_idx;  // frame map idx
-  YV12_BUFFER_CONFIG *buf;
+  struct RefCntBuffer *buf;
   struct scale_factors sf;
 } RefBuffer;
 
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 41dc30d..89e2a1b 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1068,9 +1068,16 @@
   // This function must ONLY be called when cm->fc has been initialized with
   // default probs, either by av1_setup_past_independence or after manually
   // initializing them
-  cm->frame_contexts[FRAME_CONTEXT_DEFAULTS] = *cm->fc;
+  *cm->default_frame_context = *cm->fc;
+  // TODO(jack.haughton@argondesign.com): don't think this should be necessary,
+  // but could do with fuller testing
   if (cm->large_scale_tile) {
-    for (int i = 0; i < FRAME_CONTEXTS; ++i) cm->frame_contexts[i] = *cm->fc;
+    for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
+      if (cm->frame_refs[i].buf != NULL)
+        cm->frame_refs[i].buf->frame_context = *cm->fc;
+    }
+    for (int i = 0; i < FRAME_BUFFERS; ++i)
+      cm->buffer_pool->frame_bufs[i].frame_context = *cm->fc;
   }
 }
 
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c
index e897777..858fc5d 100644
--- a/av1/common/mvref_common.c
+++ b/av1/common/mvref_common.c
@@ -346,9 +346,9 @@
   av1_set_ref_frame(rf, ref_frame);
 
   if (rf[1] == NONE_FRAME) {
-    int cur_frame_index = cm->cur_frame->cur_frame_offset;
-    int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx;
-    int frame0_index = cm->buffer_pool->frame_bufs[buf_idx_0].cur_frame_offset;
+    int cur_frame_index = cm->cur_frame->order_hint;
+    const RefCntBuffer *const buf_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].buf;
+    int frame0_index = buf_0->order_hint;
     int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info,
                                          cur_frame_index, frame0_index);
     CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[rf[0]];
@@ -381,14 +381,14 @@
     }
   } else {
     // Process compound inter mode
-    int cur_frame_index = cm->cur_frame->cur_frame_offset;
-    int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx;
-    int frame0_index = cm->buffer_pool->frame_bufs[buf_idx_0].cur_frame_offset;
+    int cur_frame_index = cm->cur_frame->order_hint;
+    const RefCntBuffer *const buf_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].buf;
+    int frame0_index = buf_0->order_hint;
 
     int cur_offset_0 = get_relative_dist(&cm->seq_params.order_hint_info,
                                          cur_frame_index, frame0_index);
-    int buf_idx_1 = cm->frame_refs[FWD_RF_OFFSET(rf[1])].idx;
-    int frame1_index = cm->buffer_pool->frame_bufs[buf_idx_1].cur_frame_offset;
+    const RefCntBuffer *const buf_1 = cm->frame_refs[FWD_RF_OFFSET(rf[1])].buf;
+    int frame1_index = buf_1->order_hint;
     int cur_offset_1 = get_relative_dist(&cm->seq_params.order_hint_info,
                                          cur_frame_index, frame1_index);
     CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[ref_frame];
@@ -864,27 +864,24 @@
 }
 
 void av1_setup_frame_buf_refs(AV1_COMMON *cm) {
-  cm->cur_frame->cur_frame_offset = cm->current_frame.order_hint;
+  cm->cur_frame->order_hint = cm->current_frame.order_hint;
 
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx;
-    if (buf_idx >= 0)
-      cm->cur_frame->ref_frame_offset[ref_frame - LAST_FRAME] =
-          cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset;
+    const RefCntBuffer *const buf = cm->frame_refs[ref_frame - LAST_FRAME].buf;
+    if (buf != NULL)
+      cm->cur_frame->ref_order_hints[ref_frame - LAST_FRAME] = buf->order_hint;
   }
 }
 
 void av1_setup_frame_sign_bias(AV1_COMMON *cm) {
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx;
-    if (cm->seq_params.order_hint_info.enable_order_hint &&
-        buf_idx != INVALID_IDX) {
-      const int ref_frame_offset =
-          cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset;
+    const RefCntBuffer *const buf = cm->frame_refs[ref_frame - LAST_FRAME].buf;
+    if (cm->seq_params.order_hint_info.enable_order_hint && buf != NULL) {
+      const int ref_order_hint = buf->order_hint;
       cm->ref_frame_sign_bias[ref_frame] =
-          (get_relative_dist(&cm->seq_params.order_hint_info, ref_frame_offset,
+          (get_relative_dist(&cm->seq_params.order_hint_info, ref_order_hint,
                              (int)cm->current_frame.order_hint) <= 0)
               ? 0
               : 1;
@@ -942,32 +939,32 @@
 
   (void)dir;
 
-  const int start_frame_idx = cm->frame_refs[FWD_RF_OFFSET(start_frame)].idx;
-  if (start_frame_idx < 0) return 0;
+  const RefCntBuffer *const start_frame_buf =
+      cm->frame_refs[FWD_RF_OFFSET(start_frame)].buf;
+  if (start_frame_buf == NULL) return 0;
 
-  if (cm->buffer_pool->frame_bufs[start_frame_idx].intra_only) return 0;
+  if (start_frame_buf->intra_only) return 0;
 
-  if (cm->buffer_pool->frame_bufs[start_frame_idx].mi_rows != cm->mi_rows ||
-      cm->buffer_pool->frame_bufs[start_frame_idx].mi_cols != cm->mi_cols)
+  if (start_frame_buf->mi_rows != cm->mi_rows ||
+      start_frame_buf->mi_cols != cm->mi_cols)
     return 0;
 
-  const int start_frame_offset =
-      cm->buffer_pool->frame_bufs[start_frame_idx].cur_frame_offset;
-  const unsigned int *const ref_frame_offsets =
-      &cm->buffer_pool->frame_bufs[start_frame_idx].ref_frame_offset[0];
-  const int cur_frame_offset = cm->cur_frame->cur_frame_offset;
+  const int start_frame_order_hint = start_frame_buf->order_hint;
+  const unsigned int *const ref_order_hints =
+      &start_frame_buf->ref_order_hints[0];
+  const int cur_order_hint = cm->cur_frame->order_hint;
   int start_to_current_frame_offset = get_relative_dist(
-      &cm->seq_params.order_hint_info, start_frame_offset, cur_frame_offset);
+      &cm->seq_params.order_hint_info, start_frame_order_hint, cur_order_hint);
 
   for (MV_REFERENCE_FRAME rf = LAST_FRAME; rf <= INTER_REFS_PER_FRAME; ++rf) {
-    ref_offset[rf] =
-        get_relative_dist(&cm->seq_params.order_hint_info, start_frame_offset,
-                          ref_frame_offsets[rf - LAST_FRAME]);
+    ref_offset[rf] = get_relative_dist(&cm->seq_params.order_hint_info,
+                                       start_frame_order_hint,
+                                       ref_order_hints[rf - LAST_FRAME]);
   }
 
   if (dir == 2) start_to_current_frame_offset = -start_to_current_frame_offset;
 
-  MV_REF *mv_ref_base = cm->buffer_pool->frame_bufs[start_frame_idx].mvs;
+  MV_REF *mv_ref_base = start_frame_buf->mvs;
   const int mvs_rows = (cm->mi_rows + 1) >> 1;
   const int mvs_cols = (cm->mi_cols + 1) >> 1;
 
@@ -1020,20 +1017,19 @@
     tpl_mvs_base[idx].ref_frame_offset = 0;
   }
 
-  const int cur_order_hint = cm->cur_frame->cur_frame_offset;
-  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
+  const int cur_order_hint = cm->cur_frame->order_hint;
 
-  int ref_buf_idx[INTER_REFS_PER_FRAME];
+  const RefCntBuffer *ref_buf[INTER_REFS_PER_FRAME];
   int ref_order_hint[INTER_REFS_PER_FRAME];
 
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
     const int ref_idx = ref_frame - LAST_FRAME;
-    const int buf_idx = cm->frame_refs[ref_idx].idx;
+    const RefCntBuffer *const buf = cm->frame_refs[ref_idx].buf;
     int order_hint = 0;
 
-    if (buf_idx >= 0) order_hint = frame_bufs[buf_idx].cur_frame_offset;
+    if (buf != NULL) order_hint = buf->order_hint;
 
-    ref_buf_idx[ref_idx] = buf_idx;
+    ref_buf[ref_idx] = buf;
     ref_order_hint[ref_idx] = order_hint;
 
     if (get_relative_dist(order_hint_info, order_hint, cur_order_hint) > 0)
@@ -1044,10 +1040,10 @@
 
   int ref_stamp = MFMV_STACK_SIZE - 1;
 
-  if (ref_buf_idx[LAST_FRAME - LAST_FRAME] >= 0) {
+  if (ref_buf[LAST_FRAME - LAST_FRAME] != NULL) {
     const int alt_of_lst_order_hint =
-        frame_bufs[ref_buf_idx[LAST_FRAME - LAST_FRAME]]
-            .ref_frame_offset[ALTREF_FRAME - LAST_FRAME];
+        ref_buf[LAST_FRAME - LAST_FRAME]
+            ->ref_order_hints[ALTREF_FRAME - LAST_FRAME];
 
     const int is_lst_overlay =
         (alt_of_lst_order_hint == ref_order_hint[GOLDEN_FRAME - LAST_FRAME]);
@@ -1073,7 +1069,7 @@
       ref_stamp >= 0)
     if (motion_field_projection(cm, ALTREF_FRAME, 0)) --ref_stamp;
 
-  if (ref_stamp >= 0 && ref_buf_idx[LAST2_FRAME - LAST_FRAME] >= 0)
+  if (ref_stamp >= 0 && ref_buf[LAST2_FRAME - LAST_FRAME] != NULL)
     if (motion_field_projection(cm, LAST2_FRAME, 2)) --ref_stamp;
 }
 
@@ -1286,32 +1282,32 @@
       cm->current_frame.reference_mode == SINGLE_REFERENCE)
     return;
 
-  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
-  const int cur_frame_offset = cm->current_frame.order_hint;
-  int ref_frame_offset[2] = { -1, INT_MAX };
+  const int cur_order_hint = cm->current_frame.order_hint;
+  int ref_order_hints[2] = { -1, INT_MAX };
   int ref_idx[2] = { INVALID_IDX, INVALID_IDX };
 
   // Identify the nearest forward and backward references.
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    const int buf_idx = cm->frame_refs[i].idx;
-    if (buf_idx == INVALID_IDX) continue;
+    const RefCntBuffer *const buf = cm->frame_refs[i].buf;
+    if (buf == NULL) continue;
 
-    const int ref_offset = frame_bufs[buf_idx].cur_frame_offset;
-    if (get_relative_dist(order_hint_info, ref_offset, cur_frame_offset) < 0) {
+    const int ref_order_hint = buf->order_hint;
+    if (get_relative_dist(order_hint_info, ref_order_hint, cur_order_hint) <
+        0) {
       // Forward reference
-      if (ref_frame_offset[0] == -1 ||
-          get_relative_dist(order_hint_info, ref_offset, ref_frame_offset[0]) >
-              0) {
-        ref_frame_offset[0] = ref_offset;
+      if (ref_order_hints[0] == -1 ||
+          get_relative_dist(order_hint_info, ref_order_hint,
+                            ref_order_hints[0]) > 0) {
+        ref_order_hints[0] = ref_order_hint;
         ref_idx[0] = i;
       }
-    } else if (get_relative_dist(order_hint_info, ref_offset,
-                                 cur_frame_offset) > 0) {
+    } else if (get_relative_dist(order_hint_info, ref_order_hint,
+                                 cur_order_hint) > 0) {
       // Backward reference
-      if (ref_frame_offset[1] == INT_MAX ||
-          get_relative_dist(order_hint_info, ref_offset, ref_frame_offset[1]) <
-              0) {
-        ref_frame_offset[1] = ref_offset;
+      if (ref_order_hints[1] == INT_MAX ||
+          get_relative_dist(order_hint_info, ref_order_hint,
+                            ref_order_hints[1]) < 0) {
+        ref_order_hints[1] = ref_order_hint;
         ref_idx[1] = i;
       }
     }
@@ -1325,24 +1321,24 @@
   } else if (ref_idx[0] != INVALID_IDX && ref_idx[1] == INVALID_IDX) {
     // == Forward prediction only ==
     // Identify the second nearest forward reference.
-    ref_frame_offset[1] = -1;
+    ref_order_hints[1] = -1;
     for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-      const int buf_idx = cm->frame_refs[i].idx;
-      if (buf_idx == INVALID_IDX) continue;
+      const RefCntBuffer *const buf = cm->frame_refs[i].buf;
+      if (buf == NULL) continue;
 
-      const int ref_offset = frame_bufs[buf_idx].cur_frame_offset;
-      if ((ref_frame_offset[0] != -1 &&
-           get_relative_dist(order_hint_info, ref_offset, ref_frame_offset[0]) <
-               0) &&
-          (ref_frame_offset[1] == -1 ||
-           get_relative_dist(order_hint_info, ref_offset, ref_frame_offset[1]) >
-               0)) {
+      const int ref_order_hint = buf->order_hint;
+      if ((ref_order_hints[0] != -1 &&
+           get_relative_dist(order_hint_info, ref_order_hint,
+                             ref_order_hints[0]) < 0) &&
+          (ref_order_hints[1] == -1 ||
+           get_relative_dist(order_hint_info, ref_order_hint,
+                             ref_order_hints[1]) > 0)) {
         // Second closest forward reference
-        ref_frame_offset[1] = ref_offset;
+        ref_order_hints[1] = ref_order_hint;
         ref_idx[1] = i;
       }
     }
-    if (ref_frame_offset[1] != -1) {
+    if (ref_order_hints[1] != -1) {
       skip_mode_info->skip_mode_allowed = 1;
       skip_mode_info->ref_frame_idx_0 = AOMMIN(ref_idx[0], ref_idx[1]);
       skip_mode_info->ref_frame_idx_1 = AOMMAX(ref_idx[0], ref_idx[1]);
@@ -1373,8 +1369,7 @@
 
   const int buf_idx = ref_info->buf_idx;
 
-  cm->frame_refs[frame_idx].idx = buf_idx;
-  cm->frame_refs[frame_idx].buf = &cm->buffer_pool->frame_bufs[buf_idx].buf;
+  cm->frame_refs[frame_idx].buf = &cm->buffer_pool->frame_bufs[buf_idx];
   cm->frame_refs[frame_idx].map_idx = ref_info->map_idx;
 }
 
@@ -1388,7 +1383,7 @@
 
   assert(cm->seq_params.order_hint_info.enable_order_hint);
   assert(cm->seq_params.order_hint_info.order_hint_bits_minus_1 >= 0);
-  const int cur_frame_offset = (int)cm->current_frame.order_hint;
+  const int cur_order_hint = (int)cm->current_frame.order_hint;
   const int cur_frame_sort_idx =
       1 << cm->seq_params.order_hint_info.order_hint_bits_minus_1;
 
@@ -1409,12 +1404,12 @@
     // TODO(zoeliu@google.com): To verify the checking on ref_count.
     if (frame_bufs[buf_idx].ref_count <= 0) continue;
 
-    const int offset = (int)frame_bufs[buf_idx].cur_frame_offset;
+    const int offset = (int)frame_bufs[buf_idx].order_hint;
     ref_frame_info[i].sort_idx =
         (offset == -1) ? -1
                        : cur_frame_sort_idx +
                              get_relative_dist(&cm->seq_params.order_hint_info,
-                                               offset, cur_frame_offset);
+                                               offset, cur_order_hint);
     assert(ref_frame_info[i].sort_idx >= -1);
 
     if (map_idx == lst_map_idx) lst_frame_sort_idx = ref_frame_info[i].sort_idx;
@@ -1437,8 +1432,8 @@
         compare_ref_frame_info);
 
   // Identify forward and backward reference frames.
-  // Forward  reference: offset < cur_frame_offset
-  // Backward reference: offset >= cur_frame_offset
+  // Forward  reference: offset < order_hint
+  // Backward reference: offset >= order_hint
   int fwd_start_idx = 0, fwd_end_idx = REF_FRAMES - 1;
 
   for (int i = 0; i < REF_FRAMES; i++) {
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index d1bd9e3..945a8dd 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -109,7 +109,20 @@
   MV_REFERENCE_FRAME ref_frame;
 } MV_REF;
 
-typedef struct {
+// FIXME(jack.haughton@argondesign.com): This enum was originally in
+// encoder/ratectrl.h, and is encoder specific. When we move to C++, this
+// should go back there and BufferPool should be templatized.
+typedef enum {
+  INTER_NORMAL = 0,
+  INTER_LOW = 1,
+  INTER_HIGH = 2,
+  GF_ARF_LOW = 3,
+  GF_ARF_STD = 4,
+  KF_STD = 5,
+  RATE_FACTOR_LEVELS = 6
+} RATE_FACTOR_LEVEL;
+
+typedef struct RefCntBuffer {
   // For a RefCntBuffer, the following are reference-holding variables:
   // - cm->ref_frame_map[]
   // - cm->new_fb_idx
@@ -123,10 +136,12 @@
   // - Total 'n' of the variables / array elements above have value 'k' (that
   // is, they are pointing to buffer at index 'k').
   // Then, pool->frame_bufs[k].ref_count = n.
+  // TODO(david.turner@argondesign.com) Check whether this helpful comment is
+  // still correct after we finish restructuring
   int ref_count;
 
-  unsigned int cur_frame_offset;
-  unsigned int ref_frame_offset[INTER_REFS_PER_FRAME];
+  unsigned int order_hint;
+  unsigned int ref_order_hints[INTER_REFS_PER_FRAME];
 
   MV_REF *mvs;
   uint8_t *seg_map;
@@ -152,6 +167,9 @@
 
   // 0 = ZERO_MV, MV
   int8_t mode_deltas[MAX_MODE_LF_DELTAS];
+
+  FRAME_CONTEXT frame_context;
+  RATE_FACTOR_LEVEL frame_rf_level;
 } RefCntBuffer;
 
 typedef struct BufferPool {
@@ -488,7 +506,7 @@
   MV_REFERENCE_FRAME comp_bwd_ref[BWD_REFS];
 
   FRAME_CONTEXT *fc;              /* this frame entropy */
-  FRAME_CONTEXT *frame_contexts;  // FRAME_CONTEXTS
+  FRAME_CONTEXT *default_frame_context;
   unsigned int frame_context_idx; /* Context to use/update */
   int fb_of_context_type[REF_FRAMES];
   int primary_ref_frame;
@@ -662,11 +680,10 @@
 
 static INLINE RefCntBuffer *get_prev_frame(const AV1_COMMON *const cm) {
   if (cm->primary_ref_frame == PRIMARY_REF_NONE ||
-      cm->frame_refs[cm->primary_ref_frame].idx == INVALID_IDX) {
+      cm->frame_refs[cm->primary_ref_frame].buf == NULL) {
     return NULL;
   } else {
-    return &cm->buffer_pool
-                ->frame_bufs[cm->frame_refs[cm->primary_ref_frame].idx];
+    return cm->frame_refs[cm->primary_ref_frame].buf;
   }
 }
 
diff --git a/av1/common/pred_common.h b/av1/common/pred_common.h
index b16cd05..721c8fe 100644
--- a/av1/common/pred_common.h
+++ b/av1/common/pred_common.h
@@ -90,16 +90,16 @@
 static INLINE int get_comp_index_context(const AV1_COMMON *cm,
                                          const MACROBLOCKD *xd) {
   MB_MODE_INFO *mbmi = xd->mi[0];
-  int bck_idx = cm->frame_refs[mbmi->ref_frame[0] - LAST_FRAME].idx;
-  int fwd_idx = cm->frame_refs[mbmi->ref_frame[1] - LAST_FRAME].idx;
+  const RefCntBuffer *const bck_buf =
+      cm->frame_refs[mbmi->ref_frame[0] - LAST_FRAME].buf;
+  const RefCntBuffer *const fwd_buf =
+      cm->frame_refs[mbmi->ref_frame[1] - LAST_FRAME].buf;
   int bck_frame_index = 0, fwd_frame_index = 0;
-  int cur_frame_index = cm->cur_frame->cur_frame_offset;
+  int cur_frame_index = cm->cur_frame->order_hint;
 
-  if (bck_idx >= 0)
-    bck_frame_index = cm->buffer_pool->frame_bufs[bck_idx].cur_frame_offset;
+  if (bck_buf != NULL) bck_frame_index = bck_buf->order_hint;
+  if (fwd_buf != NULL) fwd_frame_index = fwd_buf->order_hint;
 
-  if (fwd_idx >= 0)
-    fwd_frame_index = cm->buffer_pool->frame_bufs[fwd_idx].cur_frame_offset;
   int fwd = abs(get_relative_dist(&cm->seq_params.order_hint_info,
                                   fwd_frame_index, cur_frame_index));
   int bck = abs(get_relative_dist(&cm->seq_params.order_hint_info,
@@ -111,14 +111,14 @@
   int above_ctx = 0, left_ctx = 0;
   const int offset = (fwd == bck);
 
-  if (above_mi) {
+  if (above_mi != NULL) {
     if (has_second_ref(above_mi))
       above_ctx = above_mi->compound_idx;
     else if (above_mi->ref_frame[0] == ALTREF_FRAME)
       above_ctx = 1;
   }
 
-  if (left_mi) {
+  if (left_mi != NULL) {
     if (has_second_ref(left_mi))
       left_ctx = left_mi->compound_idx;
     else if (left_mi->ref_frame[0] == ALTREF_FRAME)
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c
index 16c4b3c..230b55d 100644
--- a/av1/common/reconinter.c
+++ b/av1/common/reconinter.c
@@ -636,18 +636,15 @@
   }
 
   *use_jnt_comp_avg = 1;
-  const int bck_idx = cm->frame_refs[mbmi->ref_frame[0] - LAST_FRAME].idx;
-  const int fwd_idx = cm->frame_refs[mbmi->ref_frame[1] - LAST_FRAME].idx;
-  const int cur_frame_index = cm->cur_frame->cur_frame_offset;
+  const RefCntBuffer *const bck_buf =
+      cm->frame_refs[mbmi->ref_frame[0] - LAST_FRAME].buf;
+  const RefCntBuffer *const fwd_buf =
+      cm->frame_refs[mbmi->ref_frame[1] - LAST_FRAME].buf;
+  const int cur_frame_index = cm->cur_frame->order_hint;
   int bck_frame_index = 0, fwd_frame_index = 0;
 
-  if (bck_idx >= 0) {
-    bck_frame_index = cm->buffer_pool->frame_bufs[bck_idx].cur_frame_offset;
-  }
-
-  if (fwd_idx >= 0) {
-    fwd_frame_index = cm->buffer_pool->frame_bufs[fwd_idx].cur_frame_offset;
-  }
+  if (bck_buf != NULL) bck_frame_index = bck_buf->order_hint;
+  if (fwd_buf != NULL) fwd_frame_index = fwd_buf->order_hint;
 
   int d0 = clamp(abs(get_relative_dist(&cm->seq_params.order_hint_info,
                                        fwd_frame_index, cur_frame_index)),
@@ -977,8 +974,8 @@
     if ((!av1_is_valid_scale(&ref_buf->sf)))
       aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM,
                          "Reference frame has invalid dimensions");
-    av1_setup_pre_planes(xd, ref, ref_buf->buf, ctxt->mi_row, above_mi_col,
-                         &ref_buf->sf, num_planes);
+    av1_setup_pre_planes(xd, ref, &ref_buf->buf->buf, ctxt->mi_row,
+                         above_mi_col, &ref_buf->sf, num_planes);
   }
 
   xd->mb_to_left_edge = 8 * MI_SIZE * (-above_mi_col);
@@ -1014,7 +1011,7 @@
     if ((!av1_is_valid_scale(&ref_buf->sf)))
       aom_internal_error(xd->error_info, AOM_CODEC_UNSUP_BITSTREAM,
                          "Reference frame has invalid dimensions");
-    av1_setup_pre_planes(xd, ref, ref_buf->buf, left_mi_row, ctxt->mi_col,
+    av1_setup_pre_planes(xd, ref, &ref_buf->buf->buf, left_mi_row, ctxt->mi_col,
                          &ref_buf->sf, num_planes);
   }
 
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 8af77e6..f9fd0a2 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -704,15 +704,15 @@
         const RefBuffer *ref_buf =
             &cm->frame_refs[this_mbmi->ref_frame[ref] - LAST_FRAME];
 
-        pd->pre[ref].buf0 =
-            (plane == 1) ? ref_buf->buf->u_buffer : ref_buf->buf->v_buffer;
+        pd->pre[ref].buf0 = (plane == 1) ? ref_buf->buf->buf.u_buffer
+                                         : ref_buf->buf->buf.v_buffer;
         pd->pre[ref].buf =
-            pd->pre[ref].buf0 + scaled_buffer_offset(pre_x, pre_y,
-                                                     ref_buf->buf->uv_stride,
-                                                     &ref_buf->sf);
-        pd->pre[ref].width = ref_buf->buf->uv_crop_width;
-        pd->pre[ref].height = ref_buf->buf->uv_crop_height;
-        pd->pre[ref].stride = ref_buf->buf->uv_stride;
+            pd->pre[ref].buf0 +
+            scaled_buffer_offset(pre_x, pre_y, ref_buf->buf->buf.uv_stride,
+                                 &ref_buf->sf);
+        pd->pre[ref].width = ref_buf->buf->buf.uv_crop_width;
+        pd->pre[ref].height = ref_buf->buf->buf.uv_crop_height;
+        pd->pre[ref].stride = ref_buf->buf->buf.uv_stride;
 
         const struct scale_factors *const sf =
             is_intrabc ? &cm->sf_identity : &ref_buf->sf;
@@ -1065,8 +1065,8 @@
       RefBuffer *ref_buf = &cm->frame_refs[frame - LAST_FRAME];
 
       xd->block_refs[ref] = ref_buf;
-      av1_setup_pre_planes(xd, ref, ref_buf->buf, mi_row, mi_col, &ref_buf->sf,
-                           num_planes);
+      av1_setup_pre_planes(xd, ref, &ref_buf->buf->buf, mi_row, mi_col,
+                           &ref_buf->sf, num_planes);
     }
   }
 
@@ -2444,7 +2444,7 @@
   int has_valid_ref_frame = 0;
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
     if (aom_rb_read_bit(rb)) {
-      YV12_BUFFER_CONFIG *const buf = cm->frame_refs[i].buf;
+      YV12_BUFFER_CONFIG *const buf = &cm->frame_refs[i].buf->buf;
       width = buf->y_crop_width;
       height = buf->y_crop_height;
       cm->render_width = buf->render_width;
@@ -2476,8 +2476,8 @@
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
     RefBuffer *const ref_frame = &cm->frame_refs[i];
     has_valid_ref_frame |=
-        valid_ref_frame_size(ref_frame->buf->y_crop_width,
-                             ref_frame->buf->y_crop_height, width, height);
+        valid_ref_frame_size(ref_frame->buf->buf.y_crop_width,
+                             ref_frame->buf->buf.y_crop_height, width, height);
   }
   if (!has_valid_ref_frame)
     aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
@@ -2485,8 +2485,8 @@
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
     RefBuffer *const ref_frame = &cm->frame_refs[i];
     if (!valid_ref_frame_img_fmt(
-            ref_frame->buf->bit_depth, ref_frame->buf->subsampling_x,
-            ref_frame->buf->subsampling_y, seq_params->bit_depth,
+            ref_frame->buf->buf.bit_depth, ref_frame->buf->buf.subsampling_x,
+            ref_frame->buf->buf.subsampling_y, seq_params->bit_depth,
             seq_params->subsampling_x, seq_params->subsampling_y))
       aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                          "Referenced frame has incompatible color format");
@@ -4721,7 +4721,6 @@
   pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1;
 
   for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    cm->frame_refs[i].idx = INVALID_IDX;
     cm->frame_refs[i].buf = NULL;
   }
 
@@ -4753,7 +4752,7 @@
   generate_next_ref_frame_map(pbi);
 
   // Reload the adapted CDFs from when we originally coded this keyframe
-  *cm->fc = cm->frame_contexts[existing_frame_idx];
+  *cm->fc = cm->frame_refs[existing_frame_idx].buf->frame_context;
 }
 
 static INLINE void reset_frame_buffers(AV1_COMMON *cm) {
@@ -4773,8 +4772,8 @@
     if (frame_bufs[i].ref_count > 0 && i != cm->new_fb_idx) {
       continue;
     }
-    frame_bufs[i].cur_frame_offset = 0;
-    av1_zero(frame_bufs[i].ref_frame_offset);
+    frame_bufs[i].order_hint = 0;
+    av1_zero(frame_bufs[i].ref_order_hints);
   }
   av1_zero_unused_internal_frame_buffers(&cm->buffer_pool->int_frame_buffers);
   unlock_buffer_pool(cm->buffer_pool);
@@ -5027,7 +5026,6 @@
       pbi->refresh_frame_flags = (1 << REF_FRAMES) - 1;
 
     for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-      cm->frame_refs[i].idx = INVALID_IDX;
       cm->frame_refs[i].buf = NULL;
     }
     if (pbi->need_resync) {
@@ -5062,13 +5060,12 @@
         seq_params->order_hint_info.enable_order_hint) {
       for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) {
         // Read order hint from bit stream
-        unsigned int frame_offset = aom_rb_read_literal(
+        unsigned int order_hint = aom_rb_read_literal(
             rb, seq_params->order_hint_info.order_hint_bits_minus_1 + 1);
         // Get buffer index
         int buf_idx = cm->ref_frame_map[ref_idx];
         assert(buf_idx < FRAME_BUFFERS);
-        if (buf_idx == -1 ||
-            frame_offset != frame_bufs[buf_idx].cur_frame_offset) {
+        if (buf_idx == -1 || order_hint != frame_bufs[buf_idx].order_hint) {
           if (buf_idx >= 0) {
             lock_buffer_pool(pool);
             decrease_ref_count(buf_idx, frame_bufs, pool);
@@ -5098,7 +5095,7 @@
           set_planes_to_neutral_grey(seq_params, &frame_bufs[buf_idx].buf, 0);
 
           cm->ref_frame_map[ref_idx] = buf_idx;
-          frame_bufs[buf_idx].cur_frame_offset = frame_offset;
+          frame_bufs[buf_idx].order_hint = order_hint;
         }
       }
     }
@@ -5169,8 +5166,7 @@
                                "Inter frame requests nonexistent reference");
 
           RefBuffer *const ref_frame = &cm->frame_refs[i];
-          ref_frame->idx = idx;
-          ref_frame->buf = &frame_bufs[idx].buf;
+          ref_frame->buf = &frame_bufs[idx];
           ref_frame->map_idx = ref;
         } else {
           ref = cm->frame_refs[i].map_idx;
@@ -5212,7 +5208,7 @@
 
     cm->prev_frame = get_prev_frame(cm);
     if (cm->primary_ref_frame != PRIMARY_REF_NONE &&
-        cm->frame_refs[cm->primary_ref_frame].idx < 0) {
+        cm->frame_refs[cm->primary_ref_frame].buf == NULL) {
       aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                          "Reference frame containing this frame's initial "
                          "frame context is unavailable.");
@@ -5227,8 +5223,8 @@
       for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
         RefBuffer *const ref_buf = &cm->frame_refs[i];
         av1_setup_scale_factors_for_frame(
-            &ref_buf->sf, ref_buf->buf->y_crop_width,
-            ref_buf->buf->y_crop_height, cm->width, cm->height);
+            &ref_buf->sf, ref_buf->buf->buf.y_crop_width,
+            ref_buf->buf->buf.y_crop_height, cm->width, cm->height);
         if ((!av1_is_valid_scale(&ref_buf->sf)))
           aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
                              "Reference frame has invalid dimensions");
@@ -5493,7 +5489,7 @@
     *p_data_end = data + uncomp_hdr_size;
     if (cm->reset_decoder_state) {
       // Use the default frame context values.
-      *cm->fc = cm->frame_contexts[FRAME_CONTEXT_DEFAULTS];
+      *cm->fc = *cm->default_frame_context;
       if (!cm->fc->initialized)
         aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                            "Uninitialized entropy context.");
@@ -5511,9 +5507,9 @@
                          cm->seq_params.subsampling_y, num_planes);
   if (cm->primary_ref_frame == PRIMARY_REF_NONE) {
     // use the default frame context values
-    *cm->fc = cm->frame_contexts[FRAME_CONTEXT_DEFAULTS];
+    *cm->fc = *cm->default_frame_context;
   } else {
-    *cm->fc = cm->frame_contexts[cm->frame_refs[cm->primary_ref_frame].idx];
+    *cm->fc = cm->frame_refs[cm->primary_ref_frame].buf->frame_context;
   }
   if (!cm->fc->initialized)
     aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
@@ -5663,6 +5659,6 @@
 
   // Non frame parallel update frame context here.
   if (!cm->large_scale_tile) {
-    cm->frame_contexts[cm->new_fb_idx] = *cm->fc;
+    cm->cur_frame->frame_context = *cm->fc;
   }
 }
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c
index 4383a43..b917648 100644
--- a/av1/decoder/decoder.c
+++ b/av1/decoder/decoder.c
@@ -90,11 +90,11 @@
 
   CHECK_MEM_ERROR(cm, cm->fc,
                   (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->fc)));
-  CHECK_MEM_ERROR(cm, cm->frame_contexts,
-                  (FRAME_CONTEXT *)aom_memalign(
-                      32, FRAME_CONTEXTS * sizeof(*cm->frame_contexts)));
+  CHECK_MEM_ERROR(
+      cm, cm->default_frame_context,
+      (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->default_frame_context)));
   memset(cm->fc, 0, sizeof(*cm->fc));
-  memset(cm->frame_contexts, 0, FRAME_CONTEXTS * sizeof(*cm->frame_contexts));
+  memset(cm->default_frame_context, 0, sizeof(*cm->default_frame_context));
 
   pbi->need_resync = 1;
   aom_once(initialize_dec);
@@ -444,7 +444,6 @@
 
     // Invalidate these references until the next frame starts.
     for (ref_index = 0; ref_index < INTER_REFS_PER_FRAME; ref_index++) {
-      cm->frame_refs[ref_index].idx = INVALID_IDX;
       cm->frame_refs[ref_index].buf = NULL;
     }
   }
@@ -468,9 +467,8 @@
     // TODO(jkoleszar): Error concealment is undefined and non-normative
     // at this point, but if it becomes so, [0] may not always be the correct
     // thing to do here.
-    if (cm->frame_refs[0].idx > 0) {
-      assert(cm->frame_refs[0].buf != NULL);
-      cm->frame_refs[0].buf->corrupted = 1;
+    if (cm->frame_refs[0].buf != NULL) {
+      cm->frame_refs[0].buf->buf.corrupted = 1;
     }
   }
 
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 22a5bc7..2121a6f 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1942,14 +1942,13 @@
 
     if (lf->mode_ref_delta_update) {
       const int prime_idx = cm->primary_ref_frame;
-      const int buf_idx =
-          prime_idx == PRIMARY_REF_NONE ? -1 : cm->frame_refs[prime_idx].idx;
+      const RefCntBuffer *const buf =
+          prime_idx == PRIMARY_REF_NONE ? NULL : cm->frame_refs[prime_idx].buf;
       int8_t last_ref_deltas[REF_FRAMES];
-      if (prime_idx == PRIMARY_REF_NONE || buf_idx < 0) {
+      if (prime_idx == PRIMARY_REF_NONE || buf == NULL) {
         av1_set_default_ref_deltas(last_ref_deltas);
       } else {
-        memcpy(last_ref_deltas, cm->buffer_pool->frame_bufs[buf_idx].ref_deltas,
-               REF_FRAMES);
+        memcpy(last_ref_deltas, buf->ref_deltas, REF_FRAMES);
       }
       for (i = 0; i < REF_FRAMES; i++) {
         const int delta = lf->ref_deltas[i];
@@ -1959,12 +1958,10 @@
       }
 
       int8_t last_mode_deltas[MAX_MODE_LF_DELTAS];
-      if (prime_idx == PRIMARY_REF_NONE || buf_idx < 0) {
+      if (prime_idx == PRIMARY_REF_NONE || buf == NULL) {
         av1_set_default_mode_deltas(last_mode_deltas);
       } else {
-        memcpy(last_mode_deltas,
-               cm->buffer_pool->frame_bufs[buf_idx].mode_deltas,
-               MAX_MODE_LF_DELTAS);
+        memcpy(last_mode_deltas, buf->mode_deltas, MAX_MODE_LF_DELTAS);
       }
       for (i = 0; i < MAX_MODE_LF_DELTAS; i++) {
         const int delta = lf->mode_deltas[i];
@@ -2893,7 +2890,7 @@
   for (int ref_idx = 0; ref_idx < INTER_REFS_PER_FRAME; ++ref_idx) {
     // Compare the buffer index between two reference frames indexed
     // respectively by the encoder and the decoder side decisions.
-    if (cm->frame_refs[ref_idx].idx != frame_refs_copy[ref_idx].idx) {
+    if (cm->frame_refs[ref_idx].buf != frame_refs_copy[ref_idx].buf) {
       cm->frame_refs_short_signaling = 0;
       break;
     }
@@ -2904,11 +2901,10 @@
   printf("***frame_refs_short_signaling=%d\n", cm->frame_refs_short_signaling);
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     printf("enc_ref(map_idx=%d, buf_idx=%d)=%d, vs. "
-        "dec_ref(map_idx=%d, buf_idx=%d)=%d\n",
+        "dec_ref(map_idx=%d)=%d\n",
         get_ref_frame_map_idx(cpi, ref_frame),
         get_ref_frame_buf_idx(cpi, ref_frame), ref_frame,
-        cm->frame_refs[ref_frame - LAST_FRAME].map_idx,
-        cm->frame_refs[ref_frame - LAST_FRAME].idx, ref_frame);
+        cm->frame_refs[ref_frame - LAST_FRAME].map_idx, ref_frame);
   }
 #endif  // 0
 
@@ -3133,7 +3129,7 @@
 
         // Write order hint to bit stream
         aom_wb_write_literal(
-            wb, frame_bufs[buf_idx].cur_frame_offset,
+            wb, frame_bufs[buf_idx].order_hint,
             seq_params->order_hint_info.order_hint_bits_minus_1 + 1);
       }
     }
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 08bb332..452a520 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5431,41 +5431,42 @@
   //     the same quality level, remove the earliest reference frame.
 
   if (total_valid_refs == INTER_REFS_PER_FRAME) {
-    unsigned int min_ref_offset = UINT_MAX;
-    unsigned int second_min_ref_offset = UINT_MAX;
+    unsigned int min_ref_order_hint = UINT_MAX;
+    unsigned int second_min_ref_order_hint = UINT_MAX;
     MV_REFERENCE_FRAME earliest_ref_frames[2] = { LAST3_FRAME, LAST2_FRAME };
-    int earliest_buf_idxes[2] = { 0 };
+    const RefCntBuffer *earliest_bufs[2] = { NULL };
 
     // Locate the earliest two reference frames except GOLDEN/ALTREF.
     for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
       // Retain GOLDEN/ALTERF
       if (ref_frame == GOLDEN_FRAME || ref_frame == ALTREF_FRAME) continue;
 
-      const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx;
-      if (buf_idx >= 0) {
-        const unsigned int ref_offset =
-            cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset;
+      const RefCntBuffer *const buf =
+          cm->frame_refs[ref_frame - LAST_FRAME].buf;
+      if (buf != NULL) {
+        const unsigned int ref_order_hint = buf->order_hint;
 
-        if (min_ref_offset == UINT_MAX) {
-          min_ref_offset = ref_offset;
+        if (min_ref_order_hint == UINT_MAX) {
+          min_ref_order_hint = ref_order_hint;
           earliest_ref_frames[0] = ref_frame;
-          earliest_buf_idxes[0] = buf_idx;
+          earliest_bufs[0] = buf;
         } else {
-          if (get_relative_dist(&cm->seq_params.order_hint_info, ref_offset,
-                                min_ref_offset) < 0) {
-            second_min_ref_offset = min_ref_offset;
+          if (get_relative_dist(&cm->seq_params.order_hint_info, ref_order_hint,
+                                min_ref_order_hint) < 0) {
+            second_min_ref_order_hint = min_ref_order_hint;
             earliest_ref_frames[1] = earliest_ref_frames[0];
-            earliest_buf_idxes[1] = earliest_buf_idxes[0];
+            earliest_bufs[1] = earliest_bufs[0];
 
-            min_ref_offset = ref_offset;
+            min_ref_order_hint = ref_order_hint;
             earliest_ref_frames[0] = ref_frame;
-            earliest_buf_idxes[0] = buf_idx;
-          } else if (second_min_ref_offset == UINT_MAX ||
+            earliest_bufs[0] = buf;
+          } else if (second_min_ref_order_hint == UINT_MAX ||
                      get_relative_dist(&cm->seq_params.order_hint_info,
-                                       ref_offset, second_min_ref_offset) < 0) {
-            second_min_ref_offset = ref_offset;
+                                       ref_order_hint,
+                                       second_min_ref_order_hint) < 0) {
+            second_min_ref_order_hint = ref_order_hint;
             earliest_ref_frames[1] = ref_frame;
-            earliest_buf_idxes[1] = buf_idx;
+            earliest_bufs[1] = buf;
           }
         }
       }
@@ -5474,7 +5475,7 @@
     RATE_FACTOR_LEVEL ref_rf_level[2];
     double ref_rf_deltas[2];
     for (int i = 0; i < 2; ++i) {
-      ref_rf_level[i] = cpi->frame_rf_level[earliest_buf_idxes[i]];
+      ref_rf_level[i] = earliest_bufs[i]->frame_rf_level;
       ref_rf_deltas[i] = rate_factor_deltas[ref_rf_level[i]];
     }
     (void)ref_rf_level;
@@ -5510,12 +5511,11 @@
 
   int one_sided_refs = 1;
   for (int ref = 0; ref < INTER_REFS_PER_FRAME; ++ref) {
-    const int buf_idx = cm->frame_refs[ref].idx;
-    if (buf_idx == INVALID_IDX) continue;
+    const RefCntBuffer *const buf = cm->frame_refs[ref].buf;
+    if (buf == NULL) continue;
 
-    const int ref_offset =
-        cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset;
-    if (get_relative_dist(&cm->seq_params.order_hint_info, ref_offset,
+    const int ref_order_hint = buf->order_hint;
+    if (get_relative_dist(&cm->seq_params.order_hint_info, ref_order_hint,
                           (int)cm->current_frame.order_hint) > 0) {
       one_sided_refs = 0;  // bwd reference
       break;
@@ -5525,17 +5525,19 @@
 }
 
 static INLINE void get_skip_mode_ref_offsets(const AV1_COMMON *cm,
-                                             int ref_offset[2]) {
+                                             int ref_order_hint[2]) {
   const SkipModeInfo *const skip_mode_info = &cm->current_frame.skip_mode_info;
-  ref_offset[0] = ref_offset[1] = 0;
+  ref_order_hint[0] = ref_order_hint[1] = 0;
   if (!skip_mode_info->skip_mode_allowed) return;
 
-  const int buf_idx_0 = cm->frame_refs[skip_mode_info->ref_frame_idx_0].idx;
-  const int buf_idx_1 = cm->frame_refs[skip_mode_info->ref_frame_idx_1].idx;
-  assert(buf_idx_0 != INVALID_IDX && buf_idx_1 != INVALID_IDX);
+  const RefCntBuffer *const buf_0 =
+      cm->frame_refs[skip_mode_info->ref_frame_idx_0].buf;
+  const RefCntBuffer *const buf_1 =
+      cm->frame_refs[skip_mode_info->ref_frame_idx_1].buf;
+  assert(buf_0 != NULL && buf_1 != NULL);
 
-  ref_offset[0] = cm->buffer_pool->frame_bufs[buf_idx_0].cur_frame_offset;
-  ref_offset[1] = cm->buffer_pool->frame_bufs[buf_idx_1].cur_frame_offset;
+  ref_order_hint[0] = buf_0->order_hint;
+  ref_order_hint[1] = buf_1->order_hint;
 }
 
 static int check_skip_mode_enabled(AV1_COMP *const cpi) {
@@ -5584,8 +5586,8 @@
       cm->global_motion[GOLDEN_FRAME].wmtype != IDENTITY) {
     return get_relative_dist(
                &cm->seq_params.order_hint_info,
-               cm->cur_frame->ref_frame_offset[ref_frame - LAST_FRAME],
-               cm->cur_frame->ref_frame_offset[GOLDEN_FRAME - LAST_FRAME]) <= 0;
+               cm->cur_frame->ref_order_hints[ref_frame - LAST_FRAME],
+               cm->cur_frame->ref_order_hints[GOLDEN_FRAME - LAST_FRAME]) <= 0;
   }
   return 0;
 }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index ce28c86..3011cfa 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -410,12 +410,12 @@
     set_sb_size(&cm->seq_params, select_sb_size(cpi));
   } else {
     if (cm->primary_ref_frame == PRIMARY_REF_NONE ||
-        cm->frame_refs[cm->primary_ref_frame].idx < 0) {
+        cm->frame_refs[cm->primary_ref_frame].buf == NULL) {
       av1_setup_past_independence(cm);
       cm->seg.update_map = 1;
       cm->seg.update_data = 1;
     } else {
-      *cm->fc = cm->frame_contexts[cm->frame_refs[cm->primary_ref_frame].idx];
+      *cm->fc = cm->frame_refs[cm->primary_ref_frame].buf->frame_context;
     }
     av1_zero(cpi->interp_filter_selected[0]);
   }
@@ -2591,11 +2591,11 @@
 
   CHECK_MEM_ERROR(cm, cm->fc,
                   (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->fc)));
-  CHECK_MEM_ERROR(cm, cm->frame_contexts,
-                  (FRAME_CONTEXT *)aom_memalign(
-                      32, FRAME_CONTEXTS * sizeof(*cm->frame_contexts)));
+  CHECK_MEM_ERROR(
+      cm, cm->default_frame_context,
+      (FRAME_CONTEXT *)aom_memalign(32, sizeof(*cm->default_frame_context)));
   memset(cm->fc, 0, sizeof(*cm->fc));
-  memset(cm->frame_contexts, 0, FRAME_CONTEXTS * sizeof(*cm->frame_contexts));
+  memset(cm->default_frame_context, 0, sizeof(*cm->default_frame_context));
 
   cpi->resize_state = 0;
   cpi->resize_avg_qp = 0;
@@ -4106,16 +4106,14 @@
     RefBuffer *const ref_buf = &cm->frame_refs[ref_frame - LAST_FRAME];
     const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
 
-    ref_buf->idx = buf_idx;
-
     if (buf_idx != INVALID_IDX) {
-      YV12_BUFFER_CONFIG *const buf = &cm->buffer_pool->frame_bufs[buf_idx].buf;
+      RefCntBuffer *const buf = &cm->buffer_pool->frame_bufs[buf_idx];
       ref_buf->buf = buf;
-      av1_setup_scale_factors_for_frame(&ref_buf->sf, buf->y_crop_width,
-                                        buf->y_crop_height, cm->width,
+      av1_setup_scale_factors_for_frame(&ref_buf->sf, buf->buf.y_crop_width,
+                                        buf->buf.y_crop_height, cm->width,
                                         cm->height);
       if (av1_is_scaled(&ref_buf->sf))
-        aom_extend_frame_borders(buf, num_planes);
+        aom_extend_frame_borders(&buf->buf, num_planes);
     } else {
       ref_buf->buf = NULL;
     }
@@ -4583,7 +4581,7 @@
     // Base q-index may have changed, so we need to assign proper default coef
     // probs before every iteration.
     if (cm->primary_ref_frame == PRIMARY_REF_NONE ||
-        cm->frame_refs[cm->primary_ref_frame].idx < 0) {
+        cm->frame_refs[cm->primary_ref_frame].buf == NULL) {
       av1_default_coef_probs(cm);
       av1_setup_frame_contexts(cm);
     }
@@ -4926,16 +4924,12 @@
       current_frame->frame_number, current_frame->order_hint, cm->show_frame,
       cm->show_existing_frame);
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx;
-    const int ref_offset =
-        (buf_idx >= 0)
-            ? (int)cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset
-            : -1;
-    printf(
-        " %d(%c-%d-%4.2f)", ref_offset,
-        (cpi->ref_frame_flags & flag_list[ref_frame]) ? 'Y' : 'N',
-        (buf_idx >= 0) ? (int)cpi->frame_rf_level[buf_idx] : -1,
-        (buf_idx >= 0) ? rate_factor_deltas[cpi->frame_rf_level[buf_idx]] : -1);
+    RefBuffer *buf = &cm->frame_refs[ref_frame - LAST_FRAME];
+    const int ref_offset = (buf->buf) ? (int)buf->buf->order_hint : -1;
+    printf(" %d(%c-%d-%4.2f)", ref_offset,
+           (cpi->ref_frame_flags & flag_list[ref_frame]) ? 'Y' : 'N',
+           (buf->buf) ? (int)buf->buf->frame_rf_level : -1,
+           (buf->buf) ? rate_factor_deltas[buf->buf->frame_rf_level] : -1);
   }
   printf(" ]\n");
 
@@ -5080,7 +5074,7 @@
 
     // Update current frame offset.
     current_frame->order_hint =
-        cm->buffer_pool->frame_bufs[cm->new_fb_idx].cur_frame_offset;
+        cm->buffer_pool->frame_bufs[cm->new_fb_idx].order_hint;
 
 #if DUMP_RECON_FRAMES == 1
     // NOTE(zoeliu): For debug - Output the filtered reconstructed video.
@@ -6795,11 +6789,11 @@
 
   if (cm->new_fb_idx == INVALID_IDX) return -1;
 
+  cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx];
   // Retain the RF_LEVEL for the current newly coded frame.
-  cpi->frame_rf_level[cm->new_fb_idx] =
+  cm->cur_frame->frame_rf_level =
       cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index];
 
-  cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx];
   cm->cur_frame->buf.buf_8bit_valid = 0;
 
   if (cpi->film_grain_table) {
@@ -6895,7 +6889,7 @@
   }
 
   if (!cm->large_scale_tile) {
-    cm->frame_contexts[cm->new_fb_idx] = *cm->fc;
+    cm->cur_frame->frame_context = *cm->fc;
   }
 
 #define EXT_TILE_DEBUG 0
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 70f3ada..d2f9b84 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -720,7 +720,6 @@
   int static_mb_pct;     // % forced skip mbs by segmentation
   int ref_frame_flags;
   int ext_ref_frame_flags;
-  RATE_FACTOR_LEVEL frame_rf_level[FRAME_BUFFERS];
 
   SPEED_FEATURES sf;
 
diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h
index 4fe2d1c..ea8975d 100644
--- a/av1/encoder/ratectrl.h
+++ b/av1/encoder/ratectrl.h
@@ -16,6 +16,7 @@
 #include "aom/aom_integer.h"
 
 #include "av1/common/blockd.h"
+#include "av1/common/onyxc_int.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -66,16 +67,6 @@
 #define MAX_GF_INTERVAL 16
 #define FIXED_GF_INTERVAL 8  // Used in some testing modes only
 
-typedef enum {
-  INTER_NORMAL = 0,
-  INTER_LOW = 1,
-  INTER_HIGH = 2,
-  GF_ARF_LOW = 3,
-  GF_ARF_STD = 4,
-  KF_STD = 5,
-  RATE_FACTOR_LEVELS = 6
-} RATE_FACTOR_LEVEL;
-
 static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = {
   1.00,  // INTER_NORMAL
   0.80,  // INTER_LOW
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index bc17fb2..aba9026 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -11139,13 +11139,13 @@
       if (ref_frame[0] == ALTREF2_FRAME || ref_frame[1] == ALTREF2_FRAME)
         if (get_relative_dist(
                 order_hint_info,
-                cm->cur_frame->ref_frame_offset[ALTREF2_FRAME - LAST_FRAME],
+                cm->cur_frame->ref_order_hints[ALTREF2_FRAME - LAST_FRAME],
                 current_frame->order_hint) < 0)
           return 1;
       if (ref_frame[0] == BWDREF_FRAME || ref_frame[1] == BWDREF_FRAME)
         if (get_relative_dist(
                 order_hint_info,
-                cm->cur_frame->ref_frame_offset[BWDREF_FRAME - LAST_FRAME],
+                cm->cur_frame->ref_order_hints[BWDREF_FRAME - LAST_FRAME],
                 current_frame->order_hint) < 0)
           return 1;
     }
@@ -11155,16 +11155,14 @@
       if (ref_frame[0] == LAST3_FRAME || ref_frame[1] == LAST3_FRAME)
         if (get_relative_dist(
                 order_hint_info,
-                cm->cur_frame->ref_frame_offset[LAST3_FRAME - LAST_FRAME],
-                cm->cur_frame->ref_frame_offset[GOLDEN_FRAME - LAST_FRAME]) <=
-            0)
+                cm->cur_frame->ref_order_hints[LAST3_FRAME - LAST_FRAME],
+                cm->cur_frame->ref_order_hints[GOLDEN_FRAME - LAST_FRAME]) <= 0)
           return 1;
       if (ref_frame[0] == LAST2_FRAME || ref_frame[1] == LAST2_FRAME)
         if (get_relative_dist(
                 order_hint_info,
-                cm->cur_frame->ref_frame_offset[LAST2_FRAME - LAST_FRAME],
-                cm->cur_frame->ref_frame_offset[GOLDEN_FRAME - LAST_FRAME]) <=
-            0)
+                cm->cur_frame->ref_order_hints[LAST2_FRAME - LAST_FRAME],
+                cm->cur_frame->ref_order_hints[GOLDEN_FRAME - LAST_FRAME]) <= 0)
           return 1;
     }
   }
@@ -11173,9 +11171,10 @@
   if ((sf->selective_ref_frame >= 2) && comp_pred && !cpi->all_one_sided_refs) {
     unsigned int ref_offsets[2];
     for (int i = 0; i < 2; ++i) {
-      const int buf_idx = cm->frame_refs[ref_frame[i] - LAST_FRAME].idx;
-      assert(buf_idx >= 0);
-      ref_offsets[i] = cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset;
+      const RefCntBuffer *const buf =
+          cm->frame_refs[ref_frame[i] - LAST_FRAME].buf;
+      assert(buf != NULL);
+      ref_offsets[i] = buf->order_hint;
     }
     if ((get_relative_dist(order_hint_info, ref_offsets[0],
                            current_frame->order_hint) <= 0 &&
diff --git a/av1/encoder/reconinter_enc.c b/av1/encoder/reconinter_enc.c
index 00f9b24..9e3c7a4 100644
--- a/av1/encoder/reconinter_enc.c
+++ b/av1/encoder/reconinter_enc.c
@@ -146,15 +146,15 @@
         const RefBuffer *ref_buf =
             &cm->frame_refs[this_mbmi->ref_frame[ref] - LAST_FRAME];
 
-        pd->pre[ref].buf0 =
-            (plane == 1) ? ref_buf->buf->u_buffer : ref_buf->buf->v_buffer;
+        pd->pre[ref].buf0 = (plane == 1) ? ref_buf->buf->buf.u_buffer
+                                         : ref_buf->buf->buf.v_buffer;
         pd->pre[ref].buf =
-            pd->pre[ref].buf0 + scaled_buffer_offset(pre_x, pre_y,
-                                                     ref_buf->buf->uv_stride,
-                                                     &ref_buf->sf);
-        pd->pre[ref].width = ref_buf->buf->uv_crop_width;
-        pd->pre[ref].height = ref_buf->buf->uv_crop_height;
-        pd->pre[ref].stride = ref_buf->buf->uv_stride;
+            pd->pre[ref].buf0 +
+            scaled_buffer_offset(pre_x, pre_y, ref_buf->buf->buf.uv_stride,
+                                 &ref_buf->sf);
+        pd->pre[ref].width = ref_buf->buf->buf.uv_crop_width;
+        pd->pre[ref].height = ref_buf->buf->buf.uv_crop_height;
+        pd->pre[ref].stride = ref_buf->buf->buf.uv_stride;
 
         const struct scale_factors *const sf =
             is_intrabc ? &cm->sf_identity : &ref_buf->sf;
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index 141b096..8166e0b 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -661,8 +661,6 @@
   int frames_to_blur_backward;
   int frames_to_blur_forward;
   RefBuffer ref_buf;
-  ref_buf.idx = INVALID_IDX;
-  ref_buf.idx = INVALID_IDX;
   ref_buf.buf = NULL;
 
   YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS] = { NULL };