diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index ec35ef2..40160b9 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -1286,11 +1286,13 @@
                                          &dst_time_stamp, &dst_end_time_stamp,
                                          !img)) {
 #if CONFIG_REFERENCE_BUFFER
-      if (cpi->common.invalid_delta_frame_id_minus1) {
-        ctx->base.err_detail = "Invalid delta_frame_id_minus1";
-        return AOM_CODEC_ERROR;
+      if (cpi->common.use_reference_buffer) {
+        if (cpi->common.invalid_delta_frame_id_minus1) {
+          ctx->base.err_detail = "Invalid delta_frame_id_minus1";
+          return AOM_CODEC_ERROR;
+        }
       }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
       if (frame_size) {
         if (ctx->pending_cx_data == 0) ctx->pending_cx_data = cx_data;
 
diff --git a/av1/av1_dx_iface.c b/av1/av1_dx_iface.c
index 5247060..3f15e0e 100644
--- a/av1/av1_dx_iface.c
+++ b/av1/av1_dx_iface.c
@@ -242,7 +242,6 @@
     si->is_kf = 1;
     intra_only_flag = 1;
     si->h = 1;
-
 #else
     int show_frame;
     int error_resilient;
@@ -273,29 +272,38 @@
 
     si->is_kf = !aom_rb_read_bit(&rb);
     show_frame = aom_rb_read_bit(&rb);
+    if (!si->is_kf) {
+      if (!show_frame) intra_only_flag = show_frame ? 0 : aom_rb_read_bit(&rb);
+    }
     error_resilient = aom_rb_read_bit(&rb);
 #if CONFIG_REFERENCE_BUFFER
-    {
+    int use_reference_buffer = 1;
+    SequenceHeader seq_params = { 0, 0, 0 };
+    if (si->is_kf) {
       /* TODO: Move outside frame loop or inside key-frame branch */
-      int frame_id_len;
-      SequenceHeader seq_params;
-      read_sequence_header(&seq_params);
+      use_reference_buffer = aom_rb_read_bit(&rb);
+      if (use_reference_buffer) {
+        read_sequence_header(&seq_params);
 #if CONFIG_EXT_TILE
-      if (large_scale_tile) seq_params.frame_id_numbers_present_flag = 0;
+        if (large_scale_tile) seq_params.frame_id_numbers_present_flag = 0;
 #endif  // CONFIG_EXT_TILE
+      }
+    }
+#endif  // CONFIG_REFERENCE_BUFFER
+#if CONFIG_REFERENCE_BUFFER
+    if (use_reference_buffer) {
+      int frame_id_len;
       if (seq_params.frame_id_numbers_present_flag) {
         frame_id_len = seq_params.frame_id_length_minus7 + 7;
         aom_rb_read_literal(&rb, frame_id_len);
       }
     }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
     if (si->is_kf) {
       if (!parse_bitdepth_colorspace_sampling(profile, &rb))
         return AOM_CODEC_UNSUP_BITSTREAM;
       av1_read_frame_size(&rb, (int *)&si->w, (int *)&si->h);
     } else {
-      intra_only_flag = show_frame ? 0 : aom_rb_read_bit(&rb);
-
       rb.bit_offset += error_resilient ? 0 : 2;  // reset_frame_context
 
       if (intra_only_flag) {
@@ -307,7 +315,7 @@
         av1_read_frame_size(&rb, (int *)&si->w, (int *)&si->h);
       }
     }
-#endif
+#endif  // CONFIG_OBU
   }
   if (is_intra_only != NULL) *is_intra_only = intra_only_flag;
   return AOM_CODEC_OK;
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index b6e319f..e162269 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -64,7 +64,7 @@
 #define FRAME_ID_NUMBERS_PRESENT_FLAG 1
 #define FRAME_ID_LENGTH_MINUS7 8         // Allows frame id up to 2^15-1
 #define DELTA_FRAME_ID_LENGTH_MINUS2 12  // Allows frame id deltas up to 2^14-1
-#endif
+#endif                                   // CONFIG_REFERENCE_BUFFER
 
 #if CONFIG_NO_FRAME_CONTEXT_SIGNALING
 #define FRAME_CONTEXTS (FRAME_BUFFERS + 1)
@@ -508,12 +508,13 @@
 #endif
   int num_tg;
 #if CONFIG_REFERENCE_BUFFER
+  int use_reference_buffer;
   int current_frame_id;
   int ref_frame_id[REF_FRAMES];
   int valid_for_referencing[REF_FRAMES];
   int refresh_mask;
   int invalid_delta_frame_id_minus1;
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 #if CONFIG_ANS && ANS_MAX_SYMBOLS
   int ans_window_size_log2;
 #endif
@@ -535,7 +536,7 @@
   int frame_id_length_minus7;
   int delta_frame_id_length_minus2;
 } SequenceHeader;
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
 // TODO(hkuang): Don't need to lock the whole pool after implementing atomic
 // frame reference count.
@@ -1276,6 +1277,10 @@
   return base_partitions[split_idx];
 }
 
+static INLINE void set_use_reference_buffer(AV1_COMMON *const cm, int use) {
+  cm->use_reference_buffer = use;
+}
+
 static INLINE void set_sb_size(AV1_COMMON *const cm, BLOCK_SIZE sb_size) {
   cm->sb_size = sb_size;
   cm->mib_size = mi_size_wide[cm->sb_size];
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index f081a40..fa5742d 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -4386,7 +4386,7 @@
   seq_params->frame_id_length_minus7 = FRAME_ID_LENGTH_MINUS7;
   seq_params->delta_frame_id_length_minus2 = DELTA_FRAME_ID_LENGTH_MINUS2;
 }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
 #if CONFIG_EXT_INTER
 static void read_compound_tools(AV1_COMMON *cm,
@@ -4591,13 +4591,6 @@
   int i, mask, ref_index = 0;
   size_t sz;
 
-#if !CONFIG_OBU
-#if CONFIG_REFERENCE_BUFFER
-  /* TODO: Move outside frame loop or inside key-frame branch */
-  read_sequence_header(&pbi->seq_params);
-#endif
-#endif
-
   cm->last_frame_type = cm->frame_type;
   cm->last_intra_only = cm->intra_only;
 
@@ -4635,15 +4628,17 @@
     const int existing_frame_idx = aom_rb_read_literal(rb, 3);
     const int frame_to_show = cm->ref_frame_map[existing_frame_idx];
 #if CONFIG_REFERENCE_BUFFER
-    if (pbi->seq_params.frame_id_numbers_present_flag) {
-      int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7;
-      int display_frame_id = aom_rb_read_literal(rb, frame_id_length);
-      /* Compare display_frame_id with ref_frame_id and check valid for
-      * referencing */
-      if (display_frame_id != cm->ref_frame_id[existing_frame_idx] ||
-          cm->valid_for_referencing[existing_frame_idx] == 0)
-        aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
-                           "Reference buffer frame ID mismatch");
+    if (cm->use_reference_buffer) {
+      if (pbi->seq_params.frame_id_numbers_present_flag) {
+        int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7;
+        int display_frame_id = aom_rb_read_literal(rb, frame_id_length);
+        /* Compare display_frame_id with ref_frame_id and check valid for
+         * referencing */
+        if (display_frame_id != cm->ref_frame_id[existing_frame_idx] ||
+            cm->valid_for_referencing[existing_frame_idx] == 0)
+          aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                             "Reference buffer frame ID mismatch");
+      }
     }
 #endif
     lock_buffer_pool(pool);
@@ -4675,54 +4670,65 @@
 
 #if !CONFIG_OBU
   cm->frame_type = (FRAME_TYPE)aom_rb_read_bit(rb);
+  cm->show_frame = aom_rb_read_bit(rb);
+  if (cm->frame_type != KEY_FRAME)
+    cm->intra_only = cm->show_frame ? 0 : aom_rb_read_bit(rb);
 #else
   cm->frame_type = (FRAME_TYPE)aom_rb_read_literal(rb, 2);  // 2 bits
+  cm->show_frame = aom_rb_read_bit(rb);
   cm->intra_only = cm->frame_type == INTRA_ONLY_FRAME;
 #endif
-  cm->show_frame = aom_rb_read_bit(rb);
   cm->error_resilient_mode = aom_rb_read_bit(rb);
 #if CONFIG_REFERENCE_BUFFER
-  if (pbi->seq_params.frame_id_numbers_present_flag) {
-    int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7;
-    int diff_len = pbi->seq_params.delta_frame_id_length_minus2 + 2;
-    int prev_frame_id = 0;
-    if (cm->frame_type != KEY_FRAME) {
-      prev_frame_id = cm->current_frame_id;
-    }
-    cm->current_frame_id = aom_rb_read_literal(rb, frame_id_length);
+  if (cm->frame_type == KEY_FRAME) {
+    cm->use_reference_buffer = aom_rb_read_bit(rb);
+#if !CONFIG_OBU
+    read_sequence_header(&pbi->seq_params);
+#endif  // !CONFIG_OBU
+  }
+  if (cm->use_reference_buffer) {
+    if (pbi->seq_params.frame_id_numbers_present_flag) {
+      int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7;
+      int diff_len = pbi->seq_params.delta_frame_id_length_minus2 + 2;
+      int prev_frame_id = 0;
+      if (cm->frame_type != KEY_FRAME) {
+        prev_frame_id = cm->current_frame_id;
+      }
+      cm->current_frame_id = aom_rb_read_literal(rb, frame_id_length);
 
-    if (cm->frame_type != KEY_FRAME) {
-      int diff_frame_id;
-      if (cm->current_frame_id > prev_frame_id) {
-        diff_frame_id = cm->current_frame_id - prev_frame_id;
-      } else {
-        diff_frame_id =
-            (1 << frame_id_length) + cm->current_frame_id - prev_frame_id;
+      if (cm->frame_type != KEY_FRAME) {
+        int diff_frame_id;
+        if (cm->current_frame_id > prev_frame_id) {
+          diff_frame_id = cm->current_frame_id - prev_frame_id;
+        } else {
+          diff_frame_id =
+              (1 << frame_id_length) + cm->current_frame_id - prev_frame_id;
+        }
+        /* Check current_frame_id for conformance */
+        if (prev_frame_id == cm->current_frame_id ||
+            diff_frame_id >= (1 << (frame_id_length - 1))) {
+          aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                             "Invalid value of current_frame_id");
+        }
       }
-      /* Check current_frame_id for conformance */
-      if (prev_frame_id == cm->current_frame_id ||
-          diff_frame_id >= (1 << (frame_id_length - 1))) {
-        aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
-                           "Invalid value of current_frame_id");
-      }
-    }
-    /* Check if some frames need to be marked as not valid for referencing */
-    for (i = 0; i < REF_FRAMES; i++) {
-      if (cm->frame_type == KEY_FRAME) {
-        cm->valid_for_referencing[i] = 0;
-      } else if (cm->current_frame_id - (1 << diff_len) > 0) {
-        if (cm->ref_frame_id[i] > cm->current_frame_id ||
-            cm->ref_frame_id[i] < cm->current_frame_id - (1 << diff_len))
+      /* Check if some frames need to be marked as not valid for referencing */
+      for (i = 0; i < REF_FRAMES; i++) {
+        if (cm->frame_type == KEY_FRAME) {
           cm->valid_for_referencing[i] = 0;
-      } else {
-        if (cm->ref_frame_id[i] > cm->current_frame_id &&
-            cm->ref_frame_id[i] <
-                (1 << frame_id_length) + cm->current_frame_id - (1 << diff_len))
-          cm->valid_for_referencing[i] = 0;
+        } else if (cm->current_frame_id - (1 << diff_len) > 0) {
+          if (cm->ref_frame_id[i] > cm->current_frame_id ||
+              cm->ref_frame_id[i] < cm->current_frame_id - (1 << diff_len))
+            cm->valid_for_referencing[i] = 0;
+        } else {
+          if (cm->ref_frame_id[i] > cm->current_frame_id &&
+              cm->ref_frame_id[i] < (1 << frame_id_length) +
+                                        cm->current_frame_id - (1 << diff_len))
+            cm->valid_for_referencing[i] = 0;
+        }
       }
     }
   }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
   if (cm->frame_type == KEY_FRAME) {
 #if !CONFIG_OBU
     read_bitdepth_colorspace_sampling(cm, rb, pbi->allow_lowbitdepth);
@@ -4763,9 +4769,6 @@
     cm->use_prev_frame_mvs = 0;
 #endif
   } else {
-#if !CONFIG_OBU
-    cm->intra_only = cm->show_frame ? 0 : aom_rb_read_bit(rb);
-#endif
     if (cm->intra_only) cm->allow_screen_content_tools = aom_rb_read_bit(rb);
 #if CONFIG_TEMPMV_SIGNALING
     if (cm->intra_only || cm->error_resilient_mode) cm->use_prev_frame_mvs = 0;
@@ -4838,22 +4841,24 @@
         cm->ref_frame_sign_bias[LAST_FRAME + i] = aom_rb_read_bit(rb);
 #endif
 #if CONFIG_REFERENCE_BUFFER
-        if (pbi->seq_params.frame_id_numbers_present_flag) {
-          int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7;
-          int diff_len = pbi->seq_params.delta_frame_id_length_minus2 + 2;
-          int delta_frame_id_minus1 = aom_rb_read_literal(rb, diff_len);
-          int ref_frame_id =
-              ((cm->current_frame_id - (delta_frame_id_minus1 + 1) +
-                (1 << frame_id_length)) %
-               (1 << frame_id_length));
-          /* Compare values derived from delta_frame_id_minus1 and
-          * refresh_frame_flags. Also, check valid for referencing */
-          if (ref_frame_id != cm->ref_frame_id[ref] ||
-              cm->valid_for_referencing[ref] == 0)
-            aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
-                               "Reference buffer frame ID mismatch");
+        if (cm->use_reference_buffer) {
+          if (pbi->seq_params.frame_id_numbers_present_flag) {
+            int frame_id_length = pbi->seq_params.frame_id_length_minus7 + 7;
+            int diff_len = pbi->seq_params.delta_frame_id_length_minus2 + 2;
+            int delta_frame_id_minus1 = aom_rb_read_literal(rb, diff_len);
+            int ref_frame_id =
+                ((cm->current_frame_id - (delta_frame_id_minus1 + 1) +
+                  (1 << frame_id_length)) %
+                 (1 << frame_id_length));
+            /* Compare values derived from delta_frame_id_minus1 and
+             * refresh_frame_flags. Also, check valid for referencing */
+            if (ref_frame_id != cm->ref_frame_id[ref] ||
+                cm->valid_for_referencing[ref] == 0)
+              aom_internal_error(&cm->error, AOM_CODEC_CORRUPT_FRAME,
+                                 "Reference buffer frame ID mismatch");
+          }
         }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
       }
 
 #if CONFIG_VAR_REFS
@@ -4914,19 +4919,21 @@
 #endif
 
 #if CONFIG_REFERENCE_BUFFER
-  if (pbi->seq_params.frame_id_numbers_present_flag) {
-    /* If bitmask is set, update reference frame id values and
-    mark frames as valid for reference */
-    int refresh_frame_flags =
-        cm->frame_type == KEY_FRAME ? 0xFF : pbi->refresh_frame_flags;
-    for (i = 0; i < REF_FRAMES; i++) {
-      if ((refresh_frame_flags >> i) & 1) {
-        cm->ref_frame_id[i] = cm->current_frame_id;
-        cm->valid_for_referencing[i] = 1;
+  if (cm->use_reference_buffer) {
+    if (pbi->seq_params.frame_id_numbers_present_flag) {
+      /* If bitmask is set, update reference frame id values and
+         mark frames as valid for reference */
+      int refresh_frame_flags =
+          cm->frame_type == KEY_FRAME ? 0xFF : pbi->refresh_frame_flags;
+      for (i = 0; i < REF_FRAMES; i++) {
+        if ((refresh_frame_flags >> i) & 1) {
+          cm->ref_frame_id[i] = cm->current_frame_id;
+          cm->valid_for_referencing[i] = 1;
+        }
       }
     }
   }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
   get_frame_new_buffer(cm)->bit_depth = cm->bit_depth;
   get_frame_new_buffer(cm)->color_space = cm->color_space;
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h
index 5128006..9a5dbb1 100644
--- a/av1/decoder/decoder.h
+++ b/av1/decoder/decoder.h
@@ -142,7 +142,7 @@
   int tg_size_bit_offset;
 #if CONFIG_REFERENCE_BUFFER
   SequenceHeader seq_params;
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 #if CONFIG_INSPECTION
   aom_inspect_cb inspect_cb;
   void *inspect_ctx;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 00961e9..c8fa96b 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -4331,7 +4331,7 @@
   seq_params->frame_id_length_minus7 = FRAME_ID_LENGTH_MINUS7;
   seq_params->delta_frame_id_length_minus2 = DELTA_FRAME_ID_LENGTH_MINUS2;
 }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
 static void write_sb_size(const AV1_COMMON *cm,
                           struct aom_write_bit_buffer *wb) {
@@ -4495,15 +4495,6 @@
   AV1_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
 
-#if CONFIG_REFERENCE_BUFFER
-  /* TODO: Move outside frame loop or inside key-frame branch */
-  write_sequence_header(
-#if CONFIG_EXT_TILE
-      cm,
-#endif  // CONFIG_EXT_TILE
-      &cpi->seq_params);
-#endif
-
   aom_wb_write_literal(wb, AOM_FRAME_MARKER, 2);
 
   write_profile(cm->profile, wb);
@@ -4531,16 +4522,18 @@
     aom_wb_write_literal(wb, cpi->existing_fb_idx_to_show, 3);
 
 #if CONFIG_REFERENCE_BUFFER
-    if (cpi->seq_params.frame_id_numbers_present_flag) {
-      int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
-      int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show];
-      aom_wb_write_literal(wb, display_frame_id, frame_id_len);
-      /* Add a zero byte to prevent emulation of superframe marker */
-      /* Same logic as when when terminating the entropy coder */
-      /* Consider to have this logic only one place */
-      aom_wb_write_literal(wb, 0, 8);
+    if (cm->use_reference_buffer) {
+      if (cpi->seq_params.frame_id_numbers_present_flag) {
+        int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
+        int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show];
+        aom_wb_write_literal(wb, display_frame_id, frame_id_len);
+        /* Add a zero byte to prevent emulation of superframe marker */
+        /* Same logic as when when terminating the entropy coder */
+        /* Consider to have this logic only one place */
+        aom_wb_write_literal(wb, 0, 8);
+      }
     }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
     return;
   } else {
@@ -4552,16 +4545,31 @@
 
   aom_wb_write_bit(wb, cm->frame_type);
   aom_wb_write_bit(wb, cm->show_frame);
+  if (cm->frame_type != KEY_FRAME)
+    if (!cm->show_frame) aom_wb_write_bit(wb, cm->intra_only);
   aom_wb_write_bit(wb, cm->error_resilient_mode);
 
+  if (cm->frame_type == KEY_FRAME) {
 #if CONFIG_REFERENCE_BUFFER
-  cm->invalid_delta_frame_id_minus1 = 0;
-  if (cpi->seq_params.frame_id_numbers_present_flag) {
-    int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
-    aom_wb_write_literal(wb, cm->current_frame_id, frame_id_len);
+    aom_wb_write_bit(wb, cm->use_reference_buffer);
+    if (cm->use_reference_buffer) {
+      write_sequence_header(
+#if CONFIG_EXT_TILE
+          cm,
+#endif  // CONFIG_EXT_TILE
+          &cpi->seq_params);
+    }
+#endif  // CONFIG_REFERENCE_BUFFER
   }
-#endif
-
+#if CONFIG_REFERENCE_BUFFER
+  if (cm->use_reference_buffer) {
+    cm->invalid_delta_frame_id_minus1 = 0;
+    if (cpi->seq_params.frame_id_numbers_present_flag) {
+      int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
+      aom_wb_write_literal(wb, cm->current_frame_id, frame_id_len);
+    }
+  }
+#endif  // CONFIG_REFERENCE_BUFFER
   if (cm->frame_type == KEY_FRAME) {
     write_bitdepth_colorspace_sampling(cm, wb);
     write_frame_size(cm, wb);
@@ -4584,7 +4592,6 @@
     }
 #endif
   } else {
-    if (!cm->show_frame) aom_wb_write_bit(wb, cm->intra_only);
     if (cm->intra_only) aom_wb_write_bit(wb, cm->allow_screen_content_tools);
 #if !CONFIG_NO_FRAME_CONTEXT_SIGNALING
     if (!cm->error_resilient_mode) {
@@ -4642,21 +4649,23 @@
                              REF_FRAMES_LOG2);
         aom_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]);
 #if CONFIG_REFERENCE_BUFFER
-        if (cpi->seq_params.frame_id_numbers_present_flag) {
-          int i = get_ref_frame_map_idx(cpi, ref_frame);
-          int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
-          int diff_len = cpi->seq_params.delta_frame_id_length_minus2 + 2;
-          int delta_frame_id_minus1 =
-              ((cm->current_frame_id - cm->ref_frame_id[i] +
-                (1 << frame_id_len)) %
-               (1 << frame_id_len)) -
-              1;
-          if (delta_frame_id_minus1 < 0 ||
-              delta_frame_id_minus1 >= (1 << diff_len))
-            cm->invalid_delta_frame_id_minus1 = 1;
-          aom_wb_write_literal(wb, delta_frame_id_minus1, diff_len);
+        if (cm->use_reference_buffer) {
+          if (cpi->seq_params.frame_id_numbers_present_flag) {
+            int i = get_ref_frame_map_idx(cpi, ref_frame);
+            int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
+            int diff_len = cpi->seq_params.delta_frame_id_length_minus2 + 2;
+            int delta_frame_id_minus1 =
+                ((cm->current_frame_id - cm->ref_frame_id[i] +
+                  (1 << frame_id_len)) %
+                 (1 << frame_id_len)) -
+                1;
+            if (delta_frame_id_minus1 < 0 ||
+                delta_frame_id_minus1 >= (1 << diff_len))
+              cm->invalid_delta_frame_id_minus1 = 1;
+            aom_wb_write_literal(wb, delta_frame_id_minus1, diff_len);
+          }
         }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
       }
 
 #if CONFIG_FRAME_SIZE
@@ -4702,8 +4711,11 @@
 #endif
 
 #if CONFIG_REFERENCE_BUFFER
-  cm->refresh_mask = cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi);
-#endif
+  if (cm->use_reference_buffer) {
+    cm->refresh_mask =
+        cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi);
+  }
+#endif  // CONFIG_REFERENCE_BUFFER
 
   if (!cm->error_resilient_mode) {
     aom_wb_write_bit(
@@ -4811,16 +4823,18 @@
     aom_wb_write_literal(wb, cpi->existing_fb_idx_to_show, 3);
 
 #if CONFIG_REFERENCE_BUFFER
-    if (cpi->seq_params.frame_id_numbers_present_flag) {
-      int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
-      int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show];
-      aom_wb_write_literal(wb, display_frame_id, frame_id_len);
-      /* Add a zero byte to prevent emulation of superframe marker */
-      /* Same logic as when when terminating the entropy coder */
-      /* Consider to have this logic only one place */
-      aom_wb_write_literal(wb, 0, 8);
+    if (cm->use_reference_buffer) {
+      if (cpi->seq_params.frame_id_numbers_present_flag) {
+        int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
+        int display_frame_id = cm->ref_frame_id[cpi->existing_fb_idx_to_show];
+        aom_wb_write_literal(wb, display_frame_id, frame_id_len);
+        /* Add a zero byte to prevent emulation of superframe marker */
+        /* Same logic as when when terminating the entropy coder */
+        /* Consider to have this logic only one place */
+        aom_wb_write_literal(wb, 0, 8);
+      }
     }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
     return;
   } else {
@@ -4839,13 +4853,20 @@
   aom_wb_write_bit(wb, cm->error_resilient_mode);
 
 #if CONFIG_REFERENCE_BUFFER
-  cm->invalid_delta_frame_id_minus1 = 0;
-  if (cpi->seq_params.frame_id_numbers_present_flag) {
-    int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
-    aom_wb_write_literal(wb, cm->current_frame_id, frame_id_len);
+  if (cm->frame_type == KEY_FRAME) {
+    aom_wb_write_bit(wb, cm->use_reference_buffer);
   }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
+#if CONFIG_REFERENCE_BUFFER
+  if (cm->use_reference_buffer) {
+    cm->invalid_delta_frame_id_minus1 = 0;
+    if (cpi->seq_params.frame_id_numbers_present_flag) {
+      int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
+      aom_wb_write_literal(wb, cm->current_frame_id, frame_id_len);
+    }
+  }
+#endif  // CONFIG_REFERENCE_BUFFER
   if (cm->frame_type == KEY_FRAME) {
     write_frame_size(cm, wb);
     write_sb_size(cm, wb);
@@ -4896,7 +4917,6 @@
     }
   } else if (cm->frame_type == INTER_FRAME) {
     MV_REFERENCE_FRAME ref_frame;
-
 #if !CONFIG_NO_FRAME_CONTEXT_SIGNALING
     if (!cm->error_resilient_mode) {
       aom_wb_write_bit(wb, cm->reset_frame_context != RESET_FRAME_CONTEXT_NONE);
@@ -4927,21 +4947,23 @@
                            REF_FRAMES_LOG2);
       aom_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]);
 #if CONFIG_REFERENCE_BUFFER
-      if (cpi->seq_params.frame_id_numbers_present_flag) {
-        int i = get_ref_frame_map_idx(cpi, ref_frame);
-        int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
-        int diff_len = cpi->seq_params.delta_frame_id_length_minus2 + 2;
-        int delta_frame_id_minus1 =
-            ((cm->current_frame_id - cm->ref_frame_id[i] +
-              (1 << frame_id_len)) %
-             (1 << frame_id_len)) -
-            1;
-        if (delta_frame_id_minus1 < 0 ||
-            delta_frame_id_minus1 >= (1 << diff_len))
-          cm->invalid_delta_frame_id_minus1 = 1;
-        aom_wb_write_literal(wb, delta_frame_id_minus1, diff_len);
+      if (cm->use_reference_buffer) {
+        if (cpi->seq_params.frame_id_numbers_present_flag) {
+          int i = get_ref_frame_map_idx(cpi, ref_frame);
+          int frame_id_len = cpi->seq_params.frame_id_length_minus7 + 7;
+          int diff_len = cpi->seq_params.delta_frame_id_length_minus2 + 2;
+          int delta_frame_id_minus1 =
+              ((cm->current_frame_id - cm->ref_frame_id[i] +
+                (1 << frame_id_len)) %
+               (1 << frame_id_len)) -
+              1;
+          if (delta_frame_id_minus1 < 0 ||
+              delta_frame_id_minus1 >= (1 << diff_len))
+            cm->invalid_delta_frame_id_minus1 = 1;
+          aom_wb_write_literal(wb, delta_frame_id_minus1, diff_len);
+        }
       }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
     }
 
 #if CONFIG_FRAME_SIZE
@@ -5008,7 +5030,7 @@
           cm->invalid_delta_frame_id_minus1 = 1;
         aom_wb_write_literal(wb, delta_frame_id_minus1, diff_len);
       }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
     }
 
 #if CONFIG_FRAME_SIZE
@@ -5048,8 +5070,11 @@
 #endif
 
 #if CONFIG_REFERENCE_BUFFER
-  cm->refresh_mask = cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi);
-#endif
+  if (cm->use_reference_buffer) {
+    cm->refresh_mask =
+        cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi);
+  }
+#endif  // CONFIG_REFERENCE_BUFFER
 
   if (!cm->error_resilient_mode) {
     aom_wb_write_bit(
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index c102e4d..eaba711 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -368,6 +368,9 @@
     cpi->refresh_alt_ref_frame = 1;
     av1_zero(cpi->interp_filter_selected);
     set_sb_size(cm, select_sb_size(cpi));
+#if CONFIG_REFERENCE_BUFFER
+    set_use_reference_buffer(cm, 0);
+#endif  // CONFIG_REFERENCE_BUFFER
   } else {
 #if CONFIG_NO_FRAME_CONTEXT_SIGNALING
     if (frame_is_intra_only(cm) || cm->error_resilient_mode ||
@@ -5397,7 +5400,7 @@
 #endif
 
 #if CONFIG_REFERENCE_BUFFER
-  {
+  if (cm->use_reference_buffer) {
     /* Non-normative definition of current_frame_id ("frame counter" with
     * wraparound) */
     const int frame_id_length = FRAME_ID_LENGTH_MINUS7 + 7;
@@ -5422,7 +5425,7 @@
           (1 << frame_id_length);
     }
   }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
 #if CONFIG_EXT_DELTA_Q
   cm->delta_q_present_flag = cpi->oxcf.deltaq_mode != NO_DELTA_Q;
@@ -5497,7 +5500,7 @@
   }
 
 #if CONFIG_REFERENCE_BUFFER
-  {
+  if (cm->use_reference_buffer) {
     int i;
     /* Update reference frame id values based on the value of refresh_mask */
     for (i = 0; i < REF_FRAMES; i++) {
@@ -5506,7 +5509,7 @@
       }
     }
   }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 
 #if DUMP_RECON_FRAMES == 1
   // NOTE(zoeliu): For debug - Output the filtered reconstructed video.
@@ -6463,10 +6466,12 @@
 #endif
 
 #if CONFIG_REFERENCE_BUFFER
-  if (*time_stamp == 0) {
-    cpi->common.current_frame_id = -1;
+  if (cm->use_reference_buffer) {
+    if (*time_stamp == 0) {
+      cpi->common.current_frame_id = -1;
+    }
   }
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 #if CONFIG_AMVR
   cpi->cur_poc++;
   if (oxcf->pass != 1 && cpi->common.allow_screen_content_tools) {
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 3930721..9a057828 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -624,7 +624,7 @@
 #endif
 #if CONFIG_REFERENCE_BUFFER
   SequenceHeader seq_params;
-#endif
+#endif  // CONFIG_REFERENCE_BUFFER
 #if CONFIG_LV_MAP
   tran_low_t *tcoeff_buf[MAX_MB_PLANE];
 #endif