diff --git a/av1/av1_dx_iface.c b/av1/av1_dx_iface.c
index 96fc077..d3ed571 100644
--- a/av1/av1_dx_iface.c
+++ b/av1/av1_dx_iface.c
@@ -44,7 +44,7 @@
   int img_avail;
   int flushed;
   int invert_tile_order;
-  int last_show_frame;  // Index of last output frame.
+  RefCntBuffer *last_show_frame;  // Last output frame buffer
   int byte_alignment;
   int skip_loop_filter;
   int skip_film_grain;
@@ -317,7 +317,6 @@
     AV1_COMMON *const cm = &frame_worker_data->pbi->common;
     BufferPool *const pool = cm->buffer_pool;
 
-    cm->new_fb_idx = INVALID_IDX;
     cm->cur_frame = NULL;
     cm->byte_alignment = ctx->byte_alignment;
     cm->skip_loop_filter = ctx->skip_loop_filter;
@@ -366,7 +365,7 @@
   int i;
   const AVxWorkerInterface *const winterface = aom_get_worker_interface();
 
-  ctx->last_show_frame = -1;
+  ctx->last_show_frame = NULL;
   ctx->next_output_worker_id = 0;
   ctx->need_resync = 1;
   ctx->num_frame_workers = 1;
@@ -527,7 +526,7 @@
 
   data2->idx = -1;
   for (int i = 0; i < REF_FRAMES; ++i)
-    if (cm->ref_frame_map[i] == cm->new_fb_idx) data2->idx = i;
+    if (cm->ref_frame_map[i] == cm->cur_frame) data2->idx = i;
   data2->buf = data;
   data2->show_existing = cm->show_existing_frame;
   return res;
@@ -549,7 +548,6 @@
   // arguments are invalid.
   if (ctx->frame_workers) {
     BufferPool *const pool = ctx->buffer_pool;
-    RefCntBuffer *const frame_bufs = pool->frame_bufs;
     lock_buffer_pool(pool);
     for (int i = 0; i < ctx->num_frame_workers; ++i) {
       AVxWorker *const worker = &ctx->frame_workers[i];
@@ -557,7 +555,7 @@
           (FrameWorkerData *)worker->data1;
       struct AV1Decoder *pbi = frame_worker_data->pbi;
       for (size_t j = 0; j < pbi->num_output_frames; j++) {
-        decrease_ref_count(pbi->output_frame_index[j], frame_bufs, pool);
+        decrease_ref_count(pbi->output_frames[j], pool);
       }
       pbi->num_output_frames = 0;
     }
@@ -694,7 +692,6 @@
           (FrameWorkerData *)worker->data1;
       AV1Decoder *const pbi = frame_worker_data->pbi;
       AV1_COMMON *const cm = &pbi->common;
-      RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
       ctx->next_output_worker_id =
           (ctx->next_output_worker_id + 1) % ctx->num_frame_workers;
       // Wait for the frame from worker thread.
@@ -707,8 +704,8 @@
         aom_film_grain_t *grain_params;
         if (av1_get_raw_frame(frame_worker_data->pbi, *index, &sd,
                               &grain_params) == 0) {
-          const int buf_idx = pbi->output_frame_index[*index];
-          ctx->last_show_frame = buf_idx;
+          RefCntBuffer *const output_frame_buf = pbi->output_frames[*index];
+          ctx->last_show_frame = output_frame_buf;
           if (ctx->need_resync) return NULL;
           yuvconfig2image(&ctx->img, sd, frame_worker_data->user_priv);
 
@@ -759,7 +756,7 @@
             ctx->img.d_w = AOMMIN(tile_width, cm->mi_cols - mi_col) * MI_SIZE;
           }
 
-          ctx->img.fb_priv = frame_bufs[buf_idx].raw_frame_buffer.priv;
+          ctx->img.fb_priv = output_frame_buf->raw_frame_buffer.priv;
           img = &ctx->img;
           img->temporal_id = cm->temporal_layer_id;
           img->spatial_id = cm->spatial_layer_id;
@@ -940,11 +937,10 @@
       FrameWorkerData *const frame_worker_data =
           (FrameWorkerData *)worker->data1;
       AV1Decoder *const pbi = frame_worker_data->pbi;
-      RefCntBuffer *const frame_bufs = pbi->common.buffer_pool->frame_bufs;
       if (pbi->seen_frame_header && pbi->num_output_frames == 0)
         return AOM_CODEC_ERROR;
-      if (ctx->last_show_frame >= 0)
-        *corrupted = frame_bufs[ctx->last_show_frame].buf.corrupted;
+      if (ctx->last_show_frame != NULL)
+        *corrupted = ctx->last_show_frame->buf.corrupted;
       return AOM_CODEC_OK;
     } else {
       return AOM_CODEC_ERROR;
diff --git a/av1/common/enums.h b/av1/common/enums.h
index eb17c58..d7c6279 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -572,8 +572,8 @@
 #define REF_FRAMES_LOG2 3
 
 // REF_FRAMES for the cm->ref_frame_map array, 1 scratch frame for the new
-// frame in cm->new_fb_idx, INTER_REFS_PER_FRAME for scaled references on the
-// encoder in the cpi->scaled_ref_idx array.
+// frame in cm->cur_frame, INTER_REFS_PER_FRAME for scaled references on the
+// encoder in the cpi->scaled_ref_buf array.
 #define FRAME_BUFFERS (REF_FRAMES + 1 + INTER_REFS_PER_FRAME)
 
 #define FWD_RF_OFFSET(ref) (ref - LAST_FRAME)
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c
index baabbcc..5bf0cb5 100644
--- a/av1/common/mvref_common.c
+++ b/av1/common/mvref_common.c
@@ -1354,9 +1354,9 @@
 }
 
 typedef struct {
-  int map_idx;   // frame map index
-  int buf_idx;   // frame buffer index
-  int sort_idx;  // index based on the offset to be used for sorting
+  int map_idx;        // frame map index
+  RefCntBuffer *buf;  // frame buffer
+  int sort_idx;       // index based on the offset to be used for sorting
 } REF_FRAME_INFO;
 
 static int compare_ref_frame_info(const void *arg_a, const void *arg_b) {
@@ -1374,18 +1374,12 @@
                                REF_FRAME_INFO *ref_info) {
   assert(frame_idx >= 0 && frame_idx < INTER_REFS_PER_FRAME);
 
-  const int buf_idx = ref_info->buf_idx;
-
-  cm->current_frame.frame_refs[frame_idx].buf =
-      &cm->buffer_pool->frame_bufs[buf_idx];
+  cm->current_frame.frame_refs[frame_idx].buf = ref_info->buf;
   cm->current_frame.frame_refs[frame_idx].map_idx = ref_info->map_idx;
 }
 
 void av1_set_frame_refs(AV1_COMMON *const cm, int lst_map_idx,
                         int gld_map_idx) {
-  BufferPool *const pool = cm->buffer_pool;
-  RefCntBuffer *const frame_bufs = pool->frame_bufs;
-
   int lst_frame_sort_idx = -1;
   int gld_frame_sort_idx = -1;
 
@@ -1404,15 +1398,14 @@
     ref_frame_info[i].map_idx = map_idx;
     ref_frame_info[i].sort_idx = -1;
 
-    const int buf_idx = cm->ref_frame_map[map_idx];
-    ref_frame_info[i].buf_idx = buf_idx;
+    RefCntBuffer *const buf = cm->ref_frame_map[map_idx];
+    ref_frame_info[i].buf = buf;
 
-    assert(buf_idx < FRAME_BUFFERS);
-    if (buf_idx < 0) continue;
+    if (buf == NULL) continue;
     // TODO(zoeliu@google.com): To verify the checking on ref_count.
-    if (frame_bufs[buf_idx].ref_count <= 0) continue;
+    if (buf->ref_count <= 0) continue;
 
-    const int offset = (int)frame_bufs[buf_idx].order_hint;
+    const int offset = (int)buf->order_hint;
     ref_frame_info[i].sort_idx =
         (offset == -1) ? -1
                        : cur_frame_sort_idx +
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index 7f3a2b6..564ef70 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -125,8 +125,8 @@
 typedef struct RefCntBuffer {
   // For a RefCntBuffer, the following are reference-holding variables:
   // - cm->ref_frame_map[]
-  // - cm->new_fb_idx
-  // - cm->scaled_ref_idx[] (encoder only)
+  // - cm->cur_frame
+  // - cm->scaled_ref_buf[] (encoder only)
   // - cm->next_ref_frame_map[] (decoder only)
   // - pbi->output_frame_index[] (decoder only)
   // With that definition, 'ref_count' is the number of reference-holding
@@ -347,20 +347,16 @@
   // TODO(hkuang): Combine this with cur_buf in macroblockd.
   RefCntBuffer *cur_frame;
 
-  // For decoder, ref_frame_map[i] maps reference type 'i' to actual index of
+  // For decoder, ref_frame_map[i] maps reference type 'i' to a pointer to
   // the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
   // For encoder, ref_frame_map[j] (where j = remapped_ref_idx[i]) maps
   // remapped reference index 'j' (that is, original reference type 'i') to
-  // actual index of the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
-  int ref_frame_map[REF_FRAMES];
+  // a pointer to the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
+  RefCntBuffer *ref_frame_map[REF_FRAMES];
 
   // Prepare ref_frame_map for the next frame.
   // Only used in frame parallel decode.
-  int next_ref_frame_map[REF_FRAMES];
-
-  // Index to the 'new' frame (i.e. the frame currently being encoded or
-  // decoded) in the buffer pool 'cm->buffer_pool'.
-  int new_fb_idx;
+  RefCntBuffer *next_ref_frame_map[REF_FRAMES];
 
   int show_frame;
   int showable_frame;  // frame can be used as show existing frame in future
@@ -582,9 +578,8 @@
 
 static INLINE YV12_BUFFER_CONFIG *get_ref_frame(AV1_COMMON *cm, int index) {
   if (index < 0 || index >= REF_FRAMES) return NULL;
-  if (cm->ref_frame_map[index] < 0) return NULL;
-  assert(cm->ref_frame_map[index] < FRAME_BUFFERS);
-  return &cm->buffer_pool->frame_bufs[cm->ref_frame_map[index]].buf;
+  if (cm->ref_frame_map[index] == NULL) return NULL;
+  return &cm->ref_frame_map[index]->buf;
 }
 
 static INLINE int get_free_fb(AV1_COMMON *cm) {
@@ -620,20 +615,20 @@
   return i;
 }
 
-// Modify 'idx_ptr' to reference the buffer at 'new_idx', and update the ref
+// Modify 'lhs_ptr' to reference the buffer at 'rhs_ptr', and update the ref
 // counts accordingly.
-static INLINE void assign_frame_buffer(RefCntBuffer *bufs, int *idx_ptr,
-                                       int new_idx) {
-  const int old_idx = *idx_ptr;
-  if (old_idx >= 0) {
-    assert(bufs[old_idx].ref_count > 0);
-    // One less reference to the buffer at 'old_idx', so decrease ref count.
-    --bufs[old_idx].ref_count;
+static INLINE void assign_frame_buffer_p(RefCntBuffer **lhs_ptr,
+                                       RefCntBuffer *rhs_ptr) {
+  RefCntBuffer *const old_ptr = *lhs_ptr;
+  if (old_ptr != NULL) {
+    assert(old_ptr->ref_count > 0);
+    // One less reference to the buffer at 'old_ptr', so decrease ref count.
+    --old_ptr->ref_count;
   }
 
-  *idx_ptr = new_idx;
-  // One more reference to the buffer at 'new_idx', so increase ref count.
-  ++bufs[new_idx].ref_count;
+  *lhs_ptr = rhs_ptr;
+  // One more reference to the buffer at 'rhs_ptr', so increase ref count.
+  ++rhs_ptr->ref_count;
 }
 
 static INLINE int frame_is_intra_only(const AV1_COMMON *const cm) {
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 7f32200..2addfd9 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -4190,20 +4190,19 @@
 
   if (!pars->update_parameters) {
     // inherit parameters from a previous reference frame
-    RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
     int film_grain_params_ref_idx = aom_rb_read_literal(rb, 3);
-    int buf_idx = cm->ref_frame_map[film_grain_params_ref_idx];
-    if (buf_idx == INVALID_IDX) {
+    RefCntBuffer *const buf = cm->ref_frame_map[film_grain_params_ref_idx];
+    if (buf == NULL) {
       aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
                          "Invalid Film grain reference idx");
     }
-    if (!frame_bufs[buf_idx].film_grain_params_present) {
+    if (!buf->film_grain_params_present) {
       aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
                          "Film grain reference parameters not available");
     }
     uint16_t random_seed = pars->random_seed;
-    *pars = frame_bufs[buf_idx].film_grain_params;  // inherit paramaters
-    pars->random_seed = random_seed;                // with new random seed
+    *pars = buf->film_grain_params;   // inherit paramaters
+    pars->random_seed = random_seed;  // with new random seed
     return;
   }
 
@@ -4663,22 +4662,20 @@
 }
 
 // Release the references to the frame buffers in cm->ref_frame_map and reset
-// all elements of cm->ref_frame_map to -1.
+// all elements of cm->ref_frame_map to NULL.
 static void reset_ref_frame_map(AV1_COMMON *const cm) {
   BufferPool *const pool = cm->buffer_pool;
-  RefCntBuffer *const frame_bufs = pool->frame_bufs;
 
   for (int i = 0; i < REF_FRAMES; i++) {
-    decrease_ref_count(cm->ref_frame_map[i], frame_bufs, pool);
+    decrease_ref_count(cm->ref_frame_map[i], pool);
+    cm->ref_frame_map[i] = NULL;
   }
-  memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map));
 }
 
 // Generate next_ref_frame_map.
 static void generate_next_ref_frame_map(AV1Decoder *const pbi) {
   AV1_COMMON *const cm = &pbi->common;
   BufferPool *const pool = cm->buffer_pool;
-  RefCntBuffer *const frame_bufs = pool->frame_bufs;
 
   lock_buffer_pool(pool);
   // cm->next_ref_frame_map holds references to frame buffers. After storing a
@@ -4687,19 +4684,19 @@
   int ref_index = 0;
   for (int mask = pbi->refresh_frame_flags; mask; mask >>= 1) {
     if (mask & 1) {
-      cm->next_ref_frame_map[ref_index] = cm->new_fb_idx;
+      cm->next_ref_frame_map[ref_index] = cm->cur_frame;
     } else {
       cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index];
     }
-    if (cm->next_ref_frame_map[ref_index] >= 0)
-      ++frame_bufs[cm->next_ref_frame_map[ref_index]].ref_count;
+    if (cm->next_ref_frame_map[ref_index] != NULL)
+      ++cm->next_ref_frame_map[ref_index]->ref_count;
     ++ref_index;
   }
 
   for (; ref_index < REF_FRAMES; ++ref_index) {
     cm->next_ref_frame_map[ref_index] = cm->ref_frame_map[ref_index];
-    if (cm->next_ref_frame_map[ref_index] >= 0)
-      ++frame_bufs[cm->next_ref_frame_map[ref_index]].ref_count;
+    if (cm->next_ref_frame_map[ref_index] != NULL)
+      ++cm->next_ref_frame_map[ref_index]->ref_count;
   }
   unlock_buffer_pool(pool);
   pbi->hold_ref_buf = 1;
@@ -4708,8 +4705,6 @@
 static void show_existing_frame_reset(AV1Decoder *const pbi,
                                       int existing_frame_idx) {
   AV1_COMMON *const cm = &pbi->common;
-  BufferPool *const pool = cm->buffer_pool;
-  RefCntBuffer *const frame_bufs = pool->frame_bufs;
 
   assert(cm->show_existing_frame);
 
@@ -4747,8 +4742,7 @@
   generate_next_ref_frame_map(pbi);
 
   // Reload the adapted CDFs from when we originally coded this keyframe
-  *cm->fc =
-      frame_bufs[cm->next_ref_frame_map[existing_frame_idx]].frame_context;
+  *cm->fc = cm->next_ref_frame_map[existing_frame_idx]->frame_context;
 }
 
 static INLINE void reset_frame_buffers(AV1_COMMON *cm) {
@@ -4763,9 +4757,9 @@
   reset_ref_frame_map(cm);
   assert(cm->cur_frame->ref_count == 1);
   for (i = 0; i < FRAME_BUFFERS; ++i) {
-    // Reset all unreferenced frame buffers. We can also reset cm->new_fb_idx
-    // because we are the sole owner of cm->new_fb_idx.
-    if (frame_bufs[i].ref_count > 0 && i != cm->new_fb_idx) {
+    // Reset all unreferenced frame buffers. We can also reset cm->cur_frame
+    // because we are the sole owner of cm->cur_frame.
+    if (frame_bufs[i].ref_count > 0 && &frame_bufs[i] != cm->cur_frame) {
       continue;
     }
     frame_bufs[i].order_hint = 0;
@@ -4814,7 +4808,7 @@
       }
       // Show an existing frame directly.
       const int existing_frame_idx = aom_rb_read_literal(rb, 3);
-      const int frame_to_show = cm->ref_frame_map[existing_frame_idx];
+      RefCntBuffer *const frame_to_show = cm->ref_frame_map[existing_frame_idx];
       if (seq_params->decoder_model_info_present_flag &&
           cm->timing_info.equal_picture_interval == 0) {
         av1_read_temporal_point_info(cm, rb);
@@ -4830,11 +4824,10 @@
                              "Reference buffer frame ID mismatch");
       }
       lock_buffer_pool(pool);
-      if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) {
+      if (frame_to_show == NULL || frame_to_show->ref_count < 1) {
         unlock_buffer_pool(pool);
         aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
-                           "Buffer %d does not contain a decoded frame",
-                           frame_to_show);
+                           "Buffer does not contain a decoded frame");
       }
       // cm->cur_frame should be the buffer referenced by the return value
       // of the get_free_fb() call in av1_receive_compressed_data(), and
@@ -4845,22 +4838,20 @@
       // decrease_ref_count(). If cm->cur_frame->raw_frame_buffer
       // has already been allocated, it will not be released by ref_cnt_fb()!
       assert(!cm->cur_frame->raw_frame_buffer.data);
-      assign_frame_buffer(frame_bufs, &cm->new_fb_idx, frame_to_show);
-      cm->cur_frame = &cm->buffer_pool->frame_bufs[cm->new_fb_idx];
-      cm->reset_decoder_state =
-          frame_bufs[frame_to_show].frame_type == KEY_FRAME;
+      assign_frame_buffer_p(&cm->cur_frame, frame_to_show);
+      cm->reset_decoder_state = frame_to_show->frame_type == KEY_FRAME;
       unlock_buffer_pool(pool);
 
       cm->lf.filter_level[0] = 0;
       cm->lf.filter_level[1] = 0;
       cm->show_frame = 1;
 
-      if (!frame_bufs[frame_to_show].showable_frame) {
+      if (!frame_to_show->showable_frame) {
         aom_merge_corrupted_flag(&xd->corrupted, 1);
       }
-      if (cm->reset_decoder_state) frame_bufs[frame_to_show].showable_frame = 0;
+      if (cm->reset_decoder_state) frame_to_show->showable_frame = 0;
 
-      cm->film_grain_params = frame_bufs[frame_to_show].film_grain_params;
+      cm->film_grain_params = frame_to_show->film_grain_params;
 
       if (cm->reset_decoder_state) {
         show_existing_frame_reset(pbi, existing_frame_idx);
@@ -5047,40 +5038,39 @@
         // Read order hint from bit stream
         unsigned int order_hint = aom_rb_read_literal(
             rb, seq_params->order_hint_info.order_hint_bits_minus_1 + 1);
-        // Get buffer index
-        int buf_idx = cm->ref_frame_map[ref_idx];
-        assert(buf_idx < FRAME_BUFFERS);
-        if (buf_idx == -1 || order_hint != frame_bufs[buf_idx].order_hint) {
-          if (buf_idx >= 0) {
+        // Get buffer
+        RefCntBuffer *buf = cm->ref_frame_map[ref_idx];
+        if (buf == NULL || order_hint != buf->order_hint) {
+          if (buf != NULL) {
             lock_buffer_pool(pool);
-            decrease_ref_count(buf_idx, frame_bufs, pool);
+            decrease_ref_count(buf, pool);
             unlock_buffer_pool(pool);
           }
           // If no corresponding buffer exists, allocate a new buffer with all
           // pixels set to neutral grey.
-          buf_idx = get_free_fb(cm);
+          int buf_idx = get_free_fb(cm);
           if (buf_idx == INVALID_IDX) {
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Unable to find free frame buffer");
           }
+          buf = &frame_bufs[buf_idx];
           lock_buffer_pool(pool);
           if (aom_realloc_frame_buffer(
-                  &frame_bufs[buf_idx].buf, seq_params->max_frame_width,
+                  &buf->buf, seq_params->max_frame_width,
                   seq_params->max_frame_height, seq_params->subsampling_x,
                   seq_params->subsampling_y, seq_params->use_highbitdepth,
                   AOM_BORDER_IN_PIXELS, cm->byte_alignment,
-                  &pool->frame_bufs[buf_idx].raw_frame_buffer, pool->get_fb_cb,
-                  pool->cb_priv)) {
-            decrease_ref_count(buf_idx, frame_bufs, pool);
+                  &buf->raw_frame_buffer, pool->get_fb_cb, pool->cb_priv)) {
+            decrease_ref_count(buf, pool);
             unlock_buffer_pool(pool);
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Failed to allocate frame buffer");
           }
           unlock_buffer_pool(pool);
-          set_planes_to_neutral_grey(seq_params, &frame_bufs[buf_idx].buf, 0);
+          set_planes_to_neutral_grey(seq_params, &buf->buf, 0);
 
-          cm->ref_frame_map[ref_idx] = buf_idx;
-          frame_bufs[buf_idx].order_hint = order_hint;
+          cm->ref_frame_map[ref_idx] = buf;
+          buf->order_hint = order_hint;
         }
       }
     }
@@ -5112,22 +5102,22 @@
       if (frame_refs_short_signaling) {
         // == LAST_FRAME ==
         const int lst_ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2);
-        const int lst_idx = cm->ref_frame_map[lst_ref];
+        const RefCntBuffer *const lst_buf = cm->ref_frame_map[lst_ref];
 
         // == GOLDEN_FRAME ==
         const int gld_ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2);
-        const int gld_idx = cm->ref_frame_map[gld_ref];
+        const RefCntBuffer *const gld_buf = cm->ref_frame_map[gld_ref];
 
         // Most of the time, streams start with a keyframe. In that case,
         // ref_frame_map will have been filled in at that point and will not
-        // contain any -1's. However, streams are explicitly allowed to start
+        // contain any NULLs. However, streams are explicitly allowed to start
         // with an intra-only frame, so long as they don't then signal a
         // reference to a slot that hasn't been set yet. That's what we are
         // checking here.
-        if (lst_idx == -1)
+        if (lst_buf == NULL)
           aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                              "Inter frame requests nonexistent reference");
-        if (gld_idx == -1)
+        if (gld_buf == NULL)
           aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                              "Inter frame requests nonexistent reference");
 
@@ -5138,7 +5128,7 @@
         int ref = 0;
         if (!frame_refs_short_signaling) {
           ref = aom_rb_read_literal(rb, REF_FRAMES_LOG2);
-          const int idx = cm->ref_frame_map[ref];
+          RefCntBuffer *const buf = cm->ref_frame_map[ref];
 
           // Most of the time, streams start with a keyframe. In that case,
           // ref_frame_map will have been filled in at that point and will not
@@ -5146,12 +5136,12 @@
           // with an intra-only frame, so long as they don't then signal a
           // reference to a slot that hasn't been set yet. That's what we are
           // checking here.
-          if (idx == -1)
+          if (buf == NULL)
             aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
                                "Inter frame requests nonexistent reference");
 
           RefBuffer *const ref_frame = &cm->current_frame.frame_refs[i];
-          ref_frame->buf = &frame_bufs[idx];
+          ref_frame->buf = buf;
           ref_frame->map_idx = ref;
         } else {
           ref = cm->current_frame.frame_refs[i].map_idx;
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c
index c7625b5..2aea93f 100644
--- a/av1/decoder/decoder.c
+++ b/av1/decoder/decoder.c
@@ -100,8 +100,10 @@
   aom_once(initialize_dec);
 
   // Initialize the references to not point to any frame buffers.
-  memset(&cm->ref_frame_map, -1, sizeof(cm->ref_frame_map));
-  memset(&cm->next_ref_frame_map, -1, sizeof(cm->next_ref_frame_map));
+  for (int i = 0; i < REF_FRAMES; i++) {
+    cm->ref_frame_map[i] = NULL;
+    cm->next_ref_frame_map[i] = NULL;
+  }
 
   cm->current_frame.frame_number = 0;
   pbi->decoding_first_frame = 1;
@@ -320,30 +322,26 @@
 static void release_frame_buffers(AV1Decoder *pbi) {
   AV1_COMMON *const cm = &pbi->common;
   BufferPool *const pool = cm->buffer_pool;
-  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   cm->cur_frame->buf.corrupted = 1;
   lock_buffer_pool(pool);
   // Release all the reference buffers in cm->next_ref_frame_map if the worker
   // thread is holding them.
   if (pbi->hold_ref_buf) {
-    int ref_index;
-    for (ref_index = 0; ref_index < REF_FRAMES; ++ref_index) {
-      const int idx = cm->next_ref_frame_map[ref_index];
-      decrease_ref_count(idx, frame_bufs, pool);
+    for (int ref_index = 0; ref_index < REF_FRAMES; ++ref_index) {
+      decrease_ref_count(cm->next_ref_frame_map[ref_index], pool);
     }
     memset(&cm->next_ref_frame_map, -1, sizeof(cm->next_ref_frame_map));
     pbi->hold_ref_buf = 0;
   }
   // Release current frame.
-  decrease_ref_count(cm->new_fb_idx, frame_bufs, pool);
+  decrease_ref_count(cm->cur_frame, pool);
   unlock_buffer_pool(pool);
-  cm->new_fb_idx = INVALID_IDX;
   cm->cur_frame = NULL;
 }
 
 // If any buffer updating is signaled it should be done here.
-// Consumes a reference to cm->new_fb_idx.
+// Consumes a reference to cm->cur_frame.
 //
 // This functions returns void. It reports failure by setting
 // cm->error.error_code.
@@ -351,7 +349,6 @@
   int ref_index = 0, mask;
   AV1_COMMON *const cm = &pbi->common;
   BufferPool *const pool = cm->buffer_pool;
-  RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
 
   if (frame_decoded) {
     lock_buffer_pool(pool);
@@ -369,10 +366,9 @@
       // cm->ref_frame_map[ref_index] before transferring the reference stored
       // in cm->next_ref_frame_map[ref_index] to cm->ref_frame_map[ref_index].
       for (mask = pbi->refresh_frame_flags; mask; mask >>= 1) {
-        const int old_idx = cm->ref_frame_map[ref_index];
-        decrease_ref_count(old_idx, frame_bufs, pool);
+        decrease_ref_count(cm->ref_frame_map[ref_index], pool);
         cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index];
-        cm->next_ref_frame_map[ref_index] = INVALID_IDX;
+        cm->next_ref_frame_map[ref_index] = NULL;
         ++ref_index;
       }
 
@@ -380,40 +376,36 @@
           !cm->show_existing_frame || cm->reset_decoder_state;
       for (; ref_index < REF_FRAMES && check_on_show_existing_frame;
            ++ref_index) {
-        const int old_idx = cm->ref_frame_map[ref_index];
-        decrease_ref_count(old_idx, frame_bufs, pool);
+        decrease_ref_count(cm->ref_frame_map[ref_index], pool);
         cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index];
-        cm->next_ref_frame_map[ref_index] = INVALID_IDX;
+        cm->next_ref_frame_map[ref_index] = NULL;
       }
     }
 
     if (cm->show_existing_frame || cm->show_frame) {
-      YV12_BUFFER_CONFIG *cur_frame = &cm->cur_frame->buf;
       if (pbi->output_all_layers) {
         // Append this frame to the output queue
         if (pbi->num_output_frames >= MAX_NUM_SPATIAL_LAYERS) {
           // We can't store the new frame anywhere, so drop it and return an
           // error
           cm->cur_frame->buf.corrupted = 1;
-          decrease_ref_count(cm->new_fb_idx, frame_bufs, pool);
+          decrease_ref_count(cm->cur_frame, pool);
           cm->error.error_code = AOM_CODEC_UNSUP_BITSTREAM;
         } else {
-          pbi->output_frames[pbi->num_output_frames] = cur_frame;
-          pbi->output_frame_index[pbi->num_output_frames] = cm->new_fb_idx;
+          pbi->output_frames[pbi->num_output_frames] = cm->cur_frame;
           pbi->num_output_frames++;
         }
       } else {
         // Replace any existing output frame
         assert(pbi->num_output_frames == 0 || pbi->num_output_frames == 1);
         if (pbi->num_output_frames > 0) {
-          decrease_ref_count(pbi->output_frame_index[0], frame_bufs, pool);
+          decrease_ref_count(pbi->output_frames[0], pool);
         }
-        pbi->output_frames[0] = cur_frame;
-        pbi->output_frame_index[0] = cm->new_fb_idx;
+        pbi->output_frames[0] = cm->cur_frame;
         pbi->num_output_frames = 1;
       }
     } else {
-      decrease_ref_count(cm->new_fb_idx, frame_bufs, pool);
+      decrease_ref_count(cm->cur_frame, pool);
     }
 
     unlock_buffer_pool(pool);
@@ -424,10 +416,9 @@
     assert(IMPLIES(!pbi->camera_frame_header_ready, !pbi->hold_ref_buf));
     // Nothing was decoded, so just drop this frame buffer
     lock_buffer_pool(pool);
-    decrease_ref_count(cm->new_fb_idx, frame_bufs, pool);
+    decrease_ref_count(cm->cur_frame, pool);
     unlock_buffer_pool(pool);
   }
-  cm->new_fb_idx = INVALID_IDX;
   cm->cur_frame = NULL;
 
   if (!pbi->camera_frame_header_ready) {
@@ -466,14 +457,14 @@
   // held.
 
   // Find a free frame buffer. Return error if can not find any.
-  cm->new_fb_idx = get_free_fb(cm);
-  if (cm->new_fb_idx == INVALID_IDX) {
+  int new_fb_idx = get_free_fb(cm);
+  if (new_fb_idx == INVALID_IDX) {
     cm->error.error_code = AOM_CODEC_MEM_ERROR;
     return 1;
   }
 
   // Assign a MV array to the frame buffer.
-  cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx];
+  cm->cur_frame = &pool->frame_bufs[new_fb_idx];
 
   if (!pbi->camera_frame_header_ready) pbi->hold_ref_buf = 0;
 
@@ -519,7 +510,7 @@
   cm->txb_count = 0;
 #endif
 
-  // Note: At this point, this function holds a reference to cm->new_fb_idx
+  // Note: At this point, this function holds a reference to cm->cur_frame
   // in the buffer pool. This reference is consumed by swap_frame_buffers().
   swap_frame_buffers(pbi, frame_decoded);
 
@@ -554,11 +545,9 @@
 // Get the frame at a particular index in the output queue
 int av1_get_raw_frame(AV1Decoder *pbi, size_t index, YV12_BUFFER_CONFIG **sd,
                       aom_film_grain_t **grain_params) {
-  RefCntBuffer *const frame_bufs = pbi->common.buffer_pool->frame_bufs;
-
   if (index >= pbi->num_output_frames) return -1;
-  *sd = pbi->output_frames[index];
-  *grain_params = &frame_bufs[pbi->output_frame_index[index]].film_grain_params;
+  *sd = &pbi->output_frames[index]->buf;
+  *grain_params = &pbi->output_frames[index]->film_grain_params;
   aom_clear_system_state();
   return 0;
 }
@@ -568,6 +557,6 @@
 int av1_get_frame_to_show(AV1Decoder *pbi, YV12_BUFFER_CONFIG *frame) {
   if (pbi->num_output_frames == 0) return -1;
 
-  *frame = *pbi->output_frames[pbi->num_output_frames - 1];
+  *frame = pbi->output_frames[pbi->num_output_frames - 1]->buf;
   return 0;
 }
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h
index 6ca28e7..72e117c 100644
--- a/av1/decoder/decoder.h
+++ b/av1/decoder/decoder.h
@@ -190,8 +190,7 @@
   // Note: The saved buffers are released at the start of the next time the
   // application calls aom_codec_decode().
   int output_all_layers;
-  YV12_BUFFER_CONFIG *output_frames[MAX_NUM_SPATIAL_LAYERS];
-  int output_frame_index[MAX_NUM_SPATIAL_LAYERS];  // Buffer pool indices
+  RefCntBuffer *output_frames[MAX_NUM_SPATIAL_LAYERS];
   size_t num_output_frames;  // How many frames are queued up so far?
 
   // In order to properly support random-access decoding, we need
@@ -283,23 +282,22 @@
 
 void av1_dec_free_cb_buf(AV1Decoder *pbi);
 
-static INLINE void decrease_ref_count(int idx, RefCntBuffer *const frame_bufs,
+static INLINE void decrease_ref_count(RefCntBuffer *const buf,
                                       BufferPool *const pool) {
-  if (idx >= 0) {
-    --frame_bufs[idx].ref_count;
+  if (buf != NULL) {
+    --buf->ref_count;
     // Reference counts should never become negative. If this assertion fails,
     // there is a bug in our reference count management.
-    assert(frame_bufs[idx].ref_count >= 0);
+    assert(buf->ref_count >= 0);
     // A worker may only get a free framebuffer index when calling get_free_fb.
     // But the raw frame buffer is not set up until we finish decoding header.
     // So if any error happens during decoding header, frame_bufs[idx] will not
     // have a valid raw frame buffer.
-    if (frame_bufs[idx].ref_count == 0 &&
-        frame_bufs[idx].raw_frame_buffer.data) {
-      pool->release_fb_cb(pool->cb_priv, &frame_bufs[idx].raw_frame_buffer);
-      frame_bufs[idx].raw_frame_buffer.data = NULL;
-      frame_bufs[idx].raw_frame_buffer.size = 0;
-      frame_bufs[idx].raw_frame_buffer.priv = NULL;
+    if (buf->ref_count == 0 && buf->raw_frame_buffer.data) {
+      pool->release_fb_cb(pool->cb_priv, &buf->raw_frame_buffer);
+      buf->raw_frame_buffer.data = NULL;
+      buf->raw_frame_buffer.size = 0;
+      buf->raw_frame_buffer.priv = NULL;
     }
   }
 }
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 8e0cdfc..58e9efe 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -2543,14 +2543,13 @@
   else
     pars->update_parameters = 1;
   if (!pars->update_parameters) {
-    RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
-    int ref_frame, ref_idx, buf_idx;
+    int ref_frame, ref_idx;
     for (ref_frame = LAST_FRAME; ref_frame < REF_FRAMES; ref_frame++) {
       ref_idx = get_ref_frame_map_idx(cpi, ref_frame);
       assert(ref_idx != INVALID_IDX);
-      buf_idx = cm->ref_frame_map[ref_idx];
-      if (frame_bufs[buf_idx].film_grain_params_present &&
-          memcmp(pars, &frame_bufs[buf_idx].film_grain_params, sizeof(*pars))) {
+      const RefCntBuffer *const buf = cm->ref_frame_map[ref_idx];
+      if (buf->film_grain_params_present &&
+          memcmp(pars, &buf->film_grain_params, sizeof(*pars))) {
         break;
       }
     }
@@ -2830,18 +2829,20 @@
   AV1_COMMON *const cm = &cpi->common;
 
   // Check whether all references are distinct frames.
-  int buf_markers[FRAME_BUFFERS] = { 0 };
-  for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-    if (buf_idx != INVALID_IDX) {
-      assert(buf_idx >= 0 && buf_idx < FRAME_BUFFERS);
-      buf_markers[buf_idx] = 1;
-    }
-  }
-
+  const RefCntBuffer *seen_bufs[FRAME_BUFFERS] = { NULL };
   int num_refs = 0;
-  for (int buf_idx = 0; buf_idx < FRAME_BUFFERS; ++buf_idx) {
-    num_refs += buf_markers[buf_idx];
+  for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
+    const RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+    if (buf != NULL) {
+      int seen = 0;
+      for (int i = 0; i < num_refs; i++) {
+        if (seen_bufs[i] == buf) {
+          seen = 1;
+          break;
+        }
+      }
+      if (!seen) seen_bufs[num_refs++] = buf;
+    }
   }
 
   // We only turn on frame_refs_short_signaling when all references are
@@ -2884,10 +2885,9 @@
   printf("\nFrame=%d: \n", cm->current_frame.frame_number);
   printf("***frame_refs_short_signaling=%d\n", frame_refs_short_signaling);
   for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    printf("enc_ref(map_idx=%d, buf_idx=%d)=%d, vs. "
+    printf("enc_ref(map_idx=%d)=%d, vs. "
         "dec_ref(map_idx=%d)=%d\n",
-        get_ref_frame_map_idx(cpi, ref_frame),
-        get_ref_frame_buf_idx(cpi, ref_frame), ref_frame,
+        get_ref_frame_map_idx(cpi, ref_frame), ref_frame,
         cm->current_frame.frame_refs[ref_frame - LAST_FRAME].map_idx,
         ref_frame);
   }
@@ -2917,16 +2917,14 @@
   }
   if (!seq_params->reduced_still_picture_hdr) {
     if (encode_show_existing_frame(cm)) {
-      RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
-      const int frame_to_show = cm->ref_frame_map[cpi->existing_fb_idx_to_show];
+      RefCntBuffer *const frame_to_show =
+          cm->ref_frame_map[cpi->existing_fb_idx_to_show];
 
-      if (frame_to_show < 0 || frame_bufs[frame_to_show].ref_count < 1) {
+      if (frame_to_show == NULL || frame_to_show->ref_count < 1) {
         aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
-                           "Buffer %d does not contain a reconstructed frame",
-                           frame_to_show);
+                           "Buffer does not contain a reconstructed frame");
       }
-      assign_frame_buffer(frame_bufs, &cm->new_fb_idx, frame_to_show);
-      cm->cur_frame = &frame_bufs[cm->new_fb_idx];
+      assign_frame_buffer_p(&cm->cur_frame, frame_to_show);
 
       aom_wb_write_bit(wb, 1);  // show_existing_frame
       aom_wb_write_literal(wb, cpi->existing_fb_idx_to_show, 3);
@@ -2941,8 +2939,7 @@
         aom_wb_write_literal(wb, display_frame_id, frame_id_len);
       }
 
-      if (cm->reset_decoder_state &&
-          frame_bufs[frame_to_show].frame_type != KEY_FRAME) {
+      if (cm->reset_decoder_state && frame_to_show->frame_type != KEY_FRAME) {
         aom_internal_error(
             &cm->error, AOM_CODEC_UNSUP_BITSTREAM,
             "show_existing_frame to reset state on KEY_FRAME only");
@@ -3096,15 +3093,14 @@
     // Write all ref frame order hints if error_resilient_mode == 1
     if (cm->error_resilient_mode &&
         seq_params->order_hint_info.enable_order_hint) {
-      RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
       for (int ref_idx = 0; ref_idx < REF_FRAMES; ref_idx++) {
         // Get buffer index
-        const int buf_idx = cm->ref_frame_map[ref_idx];
-        assert(buf_idx >= 0 && buf_idx < FRAME_BUFFERS);
+        const RefCntBuffer *buf = cm->ref_frame_map[ref_idx];
+        assert(buf != NULL);
 
         // Write order hint to bit stream
         aom_wb_write_literal(
-            wb, frame_bufs[buf_idx].order_hint,
+            wb, buf->order_hint,
             seq_params->order_hint_info.order_hint_bits_minus_1 + 1);
       }
     }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index d74edac..6817626 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2622,7 +2622,7 @@
   cpi->seq_params_locked = 0;
   cpi->partition_search_skippable_frame = 0;
   cpi->tile_data = NULL;
-  cpi->last_show_frame_buf_idx = INVALID_IDX;
+  cpi->last_show_frame_buf = NULL;
   realloc_segmentation_maps(cpi);
 
   memset(cpi->nmv_costs, 0, sizeof(cpi->nmv_costs));
@@ -3549,7 +3549,7 @@
 
   // NOTE: Save the new show frame buffer index for --test-code=warn, i.e.,
   //       for the purpose to verify no mismatch between encoder and decoder.
-  if (cm->show_frame) cpi->last_show_frame_buf_idx = cm->new_fb_idx;
+  if (cm->show_frame) cpi->last_show_frame_buf = cm->cur_frame;
 
   // In the case of show_existing frame, we will not send fresh flag
   // to decoder. Any change in the reference frame buffer can be done by
@@ -3571,8 +3571,6 @@
     cpi->rc.is_bipred_frame = 0;
   }
 
-  BufferPool *const pool = cm->buffer_pool;
-
   // At this point the new frame has been encoded.
   // If any buffer copy / swapping is signaled it should be done here.
 
@@ -3581,9 +3579,8 @@
   if ((cm->current_frame.frame_type == KEY_FRAME && cm->show_frame) ||
       frame_is_sframe(cm)) {
     for (int ref_frame = 0; ref_frame < REF_FRAMES; ++ref_frame) {
-      assign_frame_buffer(pool->frame_bufs,
-                          &cm->ref_frame_map[cpi->remapped_ref_idx[ref_frame]],
-                          cm->new_fb_idx);
+      assign_frame_buffer_p(
+          &cm->ref_frame_map[cpi->remapped_ref_idx[ref_frame]], cm->cur_frame);
     }
     return;
   }
@@ -3603,10 +3600,9 @@
     // reference instead of replacing it with overlay.
 
     if (!cpi->preserve_arf_as_gld) {
-      assign_frame_buffer(
-          pool->frame_bufs,
+      assign_frame_buffer_p(
           &cm->ref_frame_map[get_ref_frame_map_idx(cpi, ALTREF_FRAME)],
-          cm->new_fb_idx);
+          cm->cur_frame);
     }
 
     tmp = get_ref_frame_map_idx(cpi, ALTREF_FRAME);
@@ -3655,8 +3651,7 @@
     // === ALTREF_FRAME ===
     if (cpi->refresh_alt_ref_frame) {
       int arf_idx = get_ref_frame_map_idx(cpi, ALTREF_FRAME);
-      assign_frame_buffer(pool->frame_bufs, &cm->ref_frame_map[arf_idx],
-                          cm->new_fb_idx);
+      assign_frame_buffer_p(&cm->ref_frame_map[arf_idx], cm->cur_frame);
 
       memcpy(cpi->interp_filter_selected[ALTREF_FRAME],
              cpi->interp_filter_selected[0],
@@ -3665,10 +3660,9 @@
 
     // === GOLDEN_FRAME ===
     if (cpi->refresh_golden_frame) {
-      assign_frame_buffer(
-          pool->frame_bufs,
+      assign_frame_buffer_p(
           &cm->ref_frame_map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)],
-          cm->new_fb_idx);
+          cm->cur_frame);
 
       memcpy(cpi->interp_filter_selected[GOLDEN_FRAME],
              cpi->interp_filter_selected[0],
@@ -3684,17 +3678,15 @@
         // and assign the newly coded frame to BWDREF so that it always
         // keeps the nearest future frame
         int tmp = get_ref_frame_map_idx(cpi, EXTREF_FRAME);
-        assign_frame_buffer(pool->frame_bufs, &cm->ref_frame_map[tmp],
-                            cm->new_fb_idx);
+        assign_frame_buffer_p(&cm->ref_frame_map[tmp], cm->cur_frame);
 
         rshift_bwd_ref_frames(cpi);
         cpi->remapped_ref_idx[BWDREF_FRAME - 1] = tmp;
       } else {
 #endif  // USE_SYMM_MULTI_LAYER
-        assign_frame_buffer(
-            pool->frame_bufs,
+        assign_frame_buffer_p(
             &cm->ref_frame_map[get_ref_frame_map_idx(cpi, BWDREF_FRAME)],
-            cm->new_fb_idx);
+            cm->cur_frame);
 #if USE_SYMM_MULTI_LAYER
       }
 #endif
@@ -3705,10 +3697,9 @@
 
     // === ALTREF2_FRAME ===
     if (cpi->refresh_alt2_ref_frame) {
-      assign_frame_buffer(
-          pool->frame_bufs,
+      assign_frame_buffer_p(
           &cm->ref_frame_map[get_ref_frame_map_idx(cpi, ALTREF2_FRAME)],
-          cm->new_fb_idx);
+          cm->cur_frame);
 
       memcpy(cpi->interp_filter_selected[ALTREF2_FRAME],
              cpi->interp_filter_selected[0],
@@ -3719,7 +3710,7 @@
   if (cpi->refresh_last_frame) {
     // NOTE(zoeliu): We have two layers of mapping (1) from the per-frame
     // reference to the reference frame buffer virtual index; and then (2) from
-    // the virtual index to the reference frame buffer physical index:
+    // the virtual index to the reference frame buffer (RefCntBuffer):
     //
     // LAST_FRAME,                       ...,     EXTREF_FRAME
     //      |                                           |
@@ -3738,7 +3729,7 @@
     //
     // (a) To change ref_frame_map[] and have the virtual index of LAST3_FRAME
     //     to point to the newly coded frame, i.e.
-    //     ref_frame_map[lst_fb_idexes[2]] => new_fb_idx;
+    //     ref_frame_map[lst_fb_idexes[2]] => cur_frame;
     //
     // (b) To change the 1st layer mapping to have LAST_FRAME mapped to the
     //     original virtual index of LAST3_FRAME and have the other mappings
@@ -3747,10 +3738,9 @@
     //      |                      |                        |
     //      v                      v                        v
     // remapped_ref_idx[2],   remapped_ref_idx[0],     remapped_ref_idx[1]
-    assign_frame_buffer(
-        pool->frame_bufs,
+    assign_frame_buffer_p(
         &cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST3_FRAME)],
-        cm->new_fb_idx);
+        cm->cur_frame);
 
     int last3_remapped_idx = get_ref_frame_map_idx(cpi, LAST3_FRAME);
 
@@ -3797,12 +3787,11 @@
 #endif  // DUMP_REF_FRAME_IMAGES
 }
 
-static INLINE void alloc_frame_mvs(AV1_COMMON *const cm, int buffer_idx) {
-  assert(buffer_idx != INVALID_IDX);
-  RefCntBuffer *const new_fb_ptr = &cm->buffer_pool->frame_bufs[buffer_idx];
-  ensure_mv_buffer(new_fb_ptr, cm);
-  new_fb_ptr->width = cm->width;
-  new_fb_ptr->height = cm->height;
+static INLINE void alloc_frame_mvs(AV1_COMMON *const cm, RefCntBuffer *buf) {
+  assert(buf != NULL);
+  ensure_mv_buffer(buf, cm);
+  buf->width = cm->width;
+  buf->height = cm->height;
 }
 
 static void scale_references(AV1_COMP *cpi) {
@@ -3822,65 +3811,62 @@
           get_ref_frame_buffer(cpi, ref_frame);
 
       if (ref == NULL) {
-        cpi->scaled_ref_idx[ref_frame - 1] = INVALID_IDX;
+        cpi->scaled_ref_buf[ref_frame - 1] = NULL;
         continue;
       }
 
       if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height) {
-        RefCntBuffer *new_fb_ptr = NULL;
         int force_scaling = 0;
-        int new_fb = cpi->scaled_ref_idx[ref_frame - 1];
-        if (new_fb == INVALID_IDX) {
-          new_fb = get_free_fb(cm);
-          if (new_fb == INVALID_IDX)
+        RefCntBuffer *new_fb = cpi->scaled_ref_buf[ref_frame - 1];
+        if (new_fb == NULL) {
+          const int new_fb_idx = get_free_fb(cm);
+          if (new_fb_idx == INVALID_IDX) {
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Unable to find free frame buffer");
+          }
           force_scaling = 1;
+          new_fb = &pool->frame_bufs[new_fb_idx];
         }
-        new_fb_ptr = &pool->frame_bufs[new_fb];
-        if (force_scaling || new_fb_ptr->buf.y_crop_width != cm->width ||
-            new_fb_ptr->buf.y_crop_height != cm->height) {
+
+        if (force_scaling || new_fb->buf.y_crop_width != cm->width ||
+            new_fb->buf.y_crop_height != cm->height) {
           if (aom_realloc_frame_buffer(
-                  &new_fb_ptr->buf, cm->width, cm->height,
+                  &new_fb->buf, cm->width, cm->height,
                   cm->seq_params.subsampling_x, cm->seq_params.subsampling_y,
                   cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS,
                   cm->byte_alignment, NULL, NULL, NULL)) {
             if (force_scaling) {
               // Release the reference acquired in the get_free_fb() call above.
-              --new_fb_ptr->ref_count;
+              --new_fb->ref_count;
             }
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Failed to allocate frame buffer");
           }
           av1_resize_and_extend_frame(
-              ref, &new_fb_ptr->buf, (int)cm->seq_params.bit_depth, num_planes);
-          cpi->scaled_ref_idx[ref_frame - 1] = new_fb;
+              ref, &new_fb->buf, (int)cm->seq_params.bit_depth, num_planes);
+          cpi->scaled_ref_buf[ref_frame - 1] = new_fb;
           alloc_frame_mvs(cm, new_fb);
         }
       } else {
-        const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-        RefCntBuffer *const buf = &pool->frame_bufs[buf_idx];
+        RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
         buf->buf.y_crop_width = ref->y_crop_width;
         buf->buf.y_crop_height = ref->y_crop_height;
-        cpi->scaled_ref_idx[ref_frame - 1] = buf_idx;
+        cpi->scaled_ref_buf[ref_frame - 1] = buf;
         ++buf->ref_count;
       }
     } else {
-      if (cpi->oxcf.pass != 0) cpi->scaled_ref_idx[ref_frame - 1] = INVALID_IDX;
+      if (cpi->oxcf.pass != 0) cpi->scaled_ref_buf[ref_frame - 1] = NULL;
     }
   }
 }
 
 static void release_scaled_references(AV1_COMP *cpi) {
-  AV1_COMMON *cm = &cpi->common;
-  int i;
   // TODO(isbs): only refresh the necessary frames, rather than all of them
-  for (i = 0; i < INTER_REFS_PER_FRAME; ++i) {
-    const int idx = cpi->scaled_ref_idx[i];
-    if (idx != INVALID_IDX) {
-      RefCntBuffer *const buf = &cm->buffer_pool->frame_bufs[idx];
+  for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
+    RefCntBuffer *const buf = cpi->scaled_ref_buf[i];
+    if (buf != NULL) {
       --buf->ref_count;
-      cpi->scaled_ref_idx[i] = INVALID_IDX;
+      cpi->scaled_ref_buf[i] = NULL;
     }
   }
 }
@@ -3998,10 +3984,9 @@
   AV1_COMMON *const cm = &cpi->common;
   int i;
   BufferPool *const pool = cm->buffer_pool;
-  cm->new_fb_idx = INVALID_IDX;
   cm->cur_frame = NULL;
   for (i = 0; i < REF_FRAMES; ++i) {
-    cm->ref_frame_map[i] = INVALID_IDX;
+    cm->ref_frame_map[i] = NULL;
   }
   for (i = 0; i < FRAME_BUFFERS; ++i) {
     pool->frame_bufs[i].ref_count = 0;
@@ -4082,7 +4067,7 @@
     av1_set_target_rate(cpi, cm->width, cm->height);
   }
 
-  alloc_frame_mvs(cm, cm->new_fb_idx);
+  alloc_frame_mvs(cm, cm->cur_frame);
 
   // Allocate above context buffers
   if (cm->num_allocated_above_context_planes < av1_num_planes(cm) ||
@@ -4117,10 +4102,8 @@
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     RefBuffer *const ref_buf =
         &cm->current_frame.frame_refs[ref_frame - LAST_FRAME];
-    const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-
-    if (buf_idx != INVALID_IDX) {
-      RefCntBuffer *const buf = &cm->buffer_pool->frame_bufs[buf_idx];
+    RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+    if (buf != NULL) {
       ref_buf->buf = buf;
       av1_setup_scale_factors_for_frame(&ref_buf->sf, buf->buf.y_crop_width,
                                         buf->buf.y_crop_height, cm->width,
@@ -4800,7 +4783,7 @@
 }
 
 static int get_ref_frame_flags(const AV1_COMP *cpi) {
-  const int *const map = cpi->common.ref_frame_map;
+  const RefCntBuffer **map = (const RefCntBuffer **)cpi->common.ref_frame_map;
 
   // No.1 Priority: LAST_FRAME
   const int last2_is_last =
@@ -4991,12 +4974,6 @@
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame)
     printf(" %d", get_ref_frame_map_idx(cpi, ref_frame));
   printf(" ]\n");
-  printf("cm->new_fb_idx = %d\n", cm->new_fb_idx);
-  printf("cm->ref_frame_map = [");
-  for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    printf(" %d", cm->ref_frame_map[ref_frame - LAST_FRAME]);
-  }
-  printf(" ]\n");
 #endif  // 0
 
   // --- Y ---
@@ -5972,7 +5949,7 @@
 
   for (i = 0; i < FRAME_BUFFERS && frame_idx < INTER_REFS_PER_FRAME + 1; ++i) {
     if (frame_bufs[i].ref_count == 0) {
-      alloc_frame_mvs(cm, i);
+      alloc_frame_mvs(cm, &frame_bufs[i]);
       if (aom_realloc_frame_buffer(
               &frame_bufs[i].buf, cm->width, cm->height,
               seq_params->subsampling_x, seq_params->subsampling_y,
@@ -6598,14 +6575,14 @@
 
     // Find a free buffer for the new frame, releasing the reference
     // previously held.
-    if (cm->new_fb_idx != INVALID_IDX) {
-      --pool->frame_bufs[cm->new_fb_idx].ref_count;
+    if (cm->cur_frame != NULL) {
+      --cm->cur_frame->ref_count;
       cm->cur_frame = NULL;
     }
 
-    cm->new_fb_idx = get_free_fb(cm);
-    if (cm->new_fb_idx == INVALID_IDX) return -1;
-    cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx];
+    const int new_fb_idx = get_free_fb(cm);
+    if (new_fb_idx == INVALID_IDX) return -1;
+    cm->cur_frame = &pool->frame_bufs[new_fb_idx];
 
     // Clear down mmx registers
     aom_clear_system_state();
@@ -6786,15 +6763,15 @@
 
   // Find a free buffer for the new frame, releasing the reference previously
   // held.
-  if (cm->new_fb_idx != INVALID_IDX) {
-    --pool->frame_bufs[cm->new_fb_idx].ref_count;
+  if (cm->cur_frame != NULL) {
+    --cm->cur_frame->ref_count;
     cm->cur_frame = NULL;
   }
-  cm->new_fb_idx = get_free_fb(cm);
 
-  if (cm->new_fb_idx == INVALID_IDX) return -1;
+  int new_fb_idx = get_free_fb(cm);
+  if (new_fb_idx == INVALID_IDX) return -1;
 
-  cm->cur_frame = &pool->frame_bufs[cm->new_fb_idx];
+  cm->cur_frame = &pool->frame_bufs[new_fb_idx];
   // Retain the RF_LEVEL for the current newly coded frame.
   cm->cur_frame->frame_rf_level =
       cpi->twopass.gf_group.rf_level[cpi->twopass.gf_group.index];
@@ -6826,8 +6803,7 @@
   }
 
   if (cpi->oxcf.pass != 0 || frame_is_intra_only(cm) == 1) {
-    for (i = 0; i < INTER_REFS_PER_FRAME; ++i)
-      cpi->scaled_ref_idx[i] = INVALID_IDX;
+    for (i = 0; i < INTER_REFS_PER_FRAME; ++i) cpi->scaled_ref_buf[i] = NULL;
   }
 
   cm->using_qmatrix = cpi->oxcf.using_qm;
@@ -6957,10 +6933,9 @@
 }
 
 int av1_get_last_show_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *frame) {
-  if (cpi->last_show_frame_buf_idx == INVALID_IDX) return -1;
+  if (cpi->last_show_frame_buf == NULL) return -1;
 
-  *frame =
-      cpi->common.buffer_pool->frame_bufs[cpi->last_show_frame_buf_idx].buf;
+  *frame = cpi->last_show_frame_buf->buf;
   return 0;
 }
 
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index f824af0..a1daee4 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -616,14 +616,15 @@
   int previous_index;
 
   unsigned int row_mt;
-  int scaled_ref_idx[INTER_REFS_PER_FRAME];
+  RefCntBuffer *scaled_ref_buf[INTER_REFS_PER_FRAME];
 
   // For encoder, we have a two-level mapping from reference frame type to the
   // corresponding buffer in the buffer pool:
   // * 'remapped_ref_idx[i - 1]' maps reference type ‘i’ (range: LAST_FRAME ...
   // EXTREF_FRAME) to a remapped index ‘j’ (in range: 0 ... REF_FRAMES - 1)
-  // * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to actual index
-  //   of the buffer in the buffer pool ‘cm->buffer_pool.frame_bufs’.
+  // * Later, 'cm->ref_frame_map[j]' maps the remapped index ‘j’ to a pointer to
+  // the reference counted buffer structure RefCntBuffer, taken from the buffer
+  // pool cm->buffer_pool->frame_bufs.
   //
   // LAST_FRAME,                        ...,      EXTREF_FRAME
   //      |                                           |
@@ -637,7 +638,7 @@
   // have a remapped index for the same.
   int remapped_ref_idx[REF_FRAMES];
 
-  int last_show_frame_buf_idx;  // last show frame buffer index
+  RefCntBuffer *last_show_frame_buf;  // last show frame buffer
 
   // refresh_*_frame are boolean flags. If 'refresh_xyz_frame' is true, then
   // after the current frame is encoded, the XYZ reference frame gets refreshed
@@ -921,11 +922,11 @@
              : INVALID_IDX;
 }
 
-static INLINE int get_ref_frame_buf_idx(const AV1_COMP *cpi,
-                                        MV_REFERENCE_FRAME ref_frame) {
+static INLINE RefCntBuffer *get_ref_frame_buf(const AV1_COMP *cpi,
+                                              MV_REFERENCE_FRAME ref_frame) {
   const AV1_COMMON *const cm = &cpi->common;
   const int map_idx = get_ref_frame_map_idx(cpi, ref_frame);
-  return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : INVALID_IDX;
+  return (map_idx != INVALID_IDX) ? cm->ref_frame_map[map_idx] : NULL;
 }
 
 // TODO(huisu@google.com, youzhou@microsoft.com): enable hash-me for HBD.
@@ -935,28 +936,22 @@
 
 static INLINE hash_table *av1_get_ref_frame_hash_map(
     const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) {
-  const AV1_COMMON *const cm = &cpi->common;
-  const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-  return buf_idx != INVALID_IDX
-             ? &cm->buffer_pool->frame_bufs[buf_idx].hash_table
-             : NULL;
+  RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+  return buf != NULL ? &buf->hash_table : NULL;
 }
 
 static INLINE YV12_BUFFER_CONFIG *get_ref_frame_buffer(
     const AV1_COMP *cpi, MV_REFERENCE_FRAME ref_frame) {
-  const AV1_COMMON *const cm = &cpi->common;
-  const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-  return buf_idx != INVALID_IDX ? &cm->buffer_pool->frame_bufs[buf_idx].buf
-                                : NULL;
+  RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+  return buf != NULL ? &buf->buf : NULL;
 }
 
 static INLINE int enc_is_ref_frame_buf(AV1_COMP *cpi, RefCntBuffer *frame_buf) {
-  AV1_COMMON *const cm = &cpi->common;
   MV_REFERENCE_FRAME ref_frame;
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-    const int buf_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-    if (buf_idx == INVALID_IDX) continue;
-    if (frame_buf == &cm->buffer_pool->frame_bufs[buf_idx]) break;
+    const RefCntBuffer *const buf = get_ref_frame_buf(cpi, ref_frame);
+    if (buf == NULL) continue;
+    if (frame_buf == buf) break;
   }
   return (ref_frame <= ALTREF_FRAME);
 }
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index 7d3163d..15cde17 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -529,7 +529,6 @@
   const YV12_BUFFER_CONFIG *first_ref_buf = lst_yv12;
   double intra_factor;
   double brightness_factor;
-  BufferPool *const pool = cm->buffer_pool;
   const int qindex = find_fp_qindex(seq_params->bit_depth);
   const int mb_scale = mi_size_wide[BLOCK_16X16];
 
@@ -1061,8 +1060,7 @@
        ((twopass->this_frame_stats.intra_error /
          DOUBLE_DIVIDE_CHECK(twopass->this_frame_stats.coded_error)) > 2.0))) {
     if (gld_yv12 != NULL) {
-      assign_frame_buffer(
-          pool->frame_bufs,
+      assign_frame_buffer_p(
           &cm->ref_frame_map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)],
           cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST_FRAME)]);
     }
@@ -1074,17 +1072,15 @@
   aom_extend_frame_borders(new_yv12, num_planes);
 
   // The frame we just compressed now becomes the last frame.
-  assign_frame_buffer(
-      pool->frame_bufs,
+  assign_frame_buffer_p(
       &cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST_FRAME)],
-      cm->new_fb_idx);
+      cm->cur_frame);
 
   // Special case for the first frame. Copy into the GF buffer as a second
   // reference.
   if (current_frame->frame_number == 0 &&
       get_ref_frame_map_idx(cpi, GOLDEN_FRAME) != INVALID_IDX) {
-    assign_frame_buffer(
-        pool->frame_bufs,
+    assign_frame_buffer_p(
         &cm->ref_frame_map[get_ref_frame_map_idx(cpi, GOLDEN_FRAME)],
         cm->ref_frame_map[get_ref_frame_map_idx(cpi, LAST_FRAME)]);
   }
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 510bb3b..1629a6a 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -1257,13 +1257,11 @@
 
 YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const AV1_COMP *cpi,
                                              int ref_frame) {
-  const AV1_COMMON *const cm = &cpi->common;
   assert(ref_frame >= LAST_FRAME && ref_frame <= ALTREF_FRAME);
-  const int scaled_idx = cpi->scaled_ref_idx[ref_frame - 1];
-  const int ref_idx = get_ref_frame_buf_idx(cpi, ref_frame);
-  return (scaled_idx != ref_idx && scaled_idx != INVALID_IDX)
-             ? &cm->buffer_pool->frame_bufs[scaled_idx].buf
-             : NULL;
+  RefCntBuffer *const scaled_buf = cpi->scaled_ref_buf[ref_frame - 1];
+  RefCntBuffer *const ref_buf = get_ref_frame_buf(cpi, ref_frame);
+  return (scaled_buf != ref_buf && scaled_buf != NULL) ? &scaled_buf->buf
+                                                       : NULL;
 }
 
 int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x,