Remove compile guards for CONFIG_EXT_REFS

This experiment has been adopted, we can simplify the code
by dropping the associated preprocessor conditionals.

Change-Id: I4705652ac8231ee25f960e8096dc97c2aeed1d20
diff --git a/av1/encoder/bgsprite.c b/av1/encoder/bgsprite.c
index ae2cb1d..ed314e1 100644
--- a/av1/encoder/bgsprite.c
+++ b/av1/encoder/bgsprite.c
@@ -1136,7 +1136,6 @@
   if (frames_fwd > distance) frames_fwd = distance;
   frames_bwd = frames_fwd;
 
-#if CONFIG_EXT_REFS
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
   if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) {
     cpi->is_arf_filter_off[gf_group->arf_update_idx[gf_group->index]] = 1;
@@ -1145,7 +1144,6 @@
   } else {
     cpi->is_arf_filter_off[gf_group->arf_update_idx[gf_group->index]] = 0;
   }
-#endif  // CONFIG_EXT_REFS
 
   const int start_frame = distance + frames_fwd;
   const int frames_to_stitch = frames_bwd + 1 + frames_fwd;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index fecddcb..7a67092 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1082,7 +1082,6 @@
       assert(comp_ref_type == BIDIR_COMP_REFERENCE);
 #endif  // CONFIG_EXT_COMP_REFS
 
-#if CONFIG_EXT_REFS
       const int bit = (mbmi->ref_frame[0] == GOLDEN_FRAME ||
                        mbmi->ref_frame[0] == LAST3_FRAME);
 #if CONFIG_VAR_REFS
@@ -1132,12 +1131,7 @@
       }
 #endif  // CONFIG_VAR_REFS
 
-#else   // !CONFIG_EXT_REFS
-      const int bit = mbmi->ref_frame[0] == GOLDEN_FRAME;
-      WRITE_REF_BIT(bit, comp_ref_p);
-#endif  // CONFIG_EXT_REFS
     } else {
-#if CONFIG_EXT_REFS
       const int bit0 = (mbmi->ref_frame[0] <= ALTREF_FRAME &&
                         mbmi->ref_frame[0] >= BWDREF_FRAME);
 #if CONFIG_VAR_REFS
@@ -1197,15 +1191,6 @@
 #endif  // CONFIG_VAR_REFS
         }
       }
-#else   // !CONFIG_EXT_REFS
-      const int bit0 = mbmi->ref_frame[0] != LAST_FRAME;
-      WRITE_REF_BIT(bit0, single_ref_p1);
-
-      if (bit0) {
-        const int bit1 = mbmi->ref_frame[0] != GOLDEN_FRAME;
-        WRITE_REF_BIT(bit1, single_ref_p2);
-      }
-#endif  // CONFIG_EXT_REFS
     }
   }
 }
@@ -3821,7 +3806,6 @@
 #endif  // CONFIG_LOOPFILTERING_ACROSS_TILES
 }
 
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
 static int get_refresh_mask_gf16(AV1_COMP *cpi) {
   int refresh_mask = 0;
@@ -3836,11 +3820,9 @@
   return refresh_mask;
 }
 #endif  // USE_GF16_MULTI_LAYER
-#endif  // CONFIG_EXT_REFS
 
 static int get_refresh_mask(AV1_COMP *cpi) {
   int refresh_mask = 0;
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
   if (cpi->rc.baseline_gf_interval == 16) return get_refresh_mask_gf16(cpi);
 #endif  // USE_GF16_MULTI_LAYER
@@ -3858,9 +3840,6 @@
 
   refresh_mask |= (cpi->refresh_bwd_ref_frame << cpi->bwd_fb_idx);
   refresh_mask |= (cpi->refresh_alt2_ref_frame << cpi->alt2_fb_idx);
-#else   // !CONFIG_EXT_REFS
-  refresh_mask |= (cpi->refresh_last_frame << cpi->lst_fb_idx);
-#endif  // CONFIG_EXT_REFS
 
   if (av1_preserve_existing_gf(cpi)) {
     // We have decided to preserve the previously existing golden frame as our
@@ -3875,15 +3854,7 @@
     // (like RTC/temporal scalability).
     return refresh_mask | (cpi->refresh_golden_frame << cpi->alt_fb_idx);
   } else {
-#if CONFIG_EXT_REFS
     const int arf_idx = cpi->alt_fb_idx;
-#else   // !CONFIG_EXT_REFS
-    int arf_idx = cpi->alt_fb_idx;
-    if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) {
-      const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
-      arf_idx = gf_group->arf_update_idx[gf_group->index];
-    }
-#endif  // CONFIG_EXT_REFS
     return refresh_mask | (cpi->refresh_golden_frame << cpi->gld_fb_idx) |
            (cpi->refresh_alt_ref_frame << arf_idx);
   }
@@ -4085,12 +4056,10 @@
 #endif  // CONFIG_EXT_TILE
     write_uncompressed_header_frame(cpi, &wb);
 
-#if CONFIG_EXT_REFS
     if (cm->show_existing_frame) {
       total_size = aom_wb_bytes_written(&wb);
       return (uint32_t)total_size;
     }
-#endif  // CONFIG_EXT_REFS
 
     // Write the tile length code
     tile_size_bytes_wb = wb;
@@ -4578,7 +4547,6 @@
   aom_wb_write_literal(wb, cm->large_scale_tile, 1);
 #endif  // CONFIG_EXT_TILE
 
-#if CONFIG_EXT_REFS
   // NOTE: By default all coded frames to be used as a reference
   cm->is_reference_frame = 1;
 
@@ -4610,11 +4578,8 @@
 
     return;
   } else {
-#endif                        // CONFIG_EXT_REFS
     aom_wb_write_bit(wb, 0);  // show_existing_frame
-#if CONFIG_EXT_REFS
   }
-#endif  // CONFIG_EXT_REFS
 
   aom_wb_write_bit(wb, cm->frame_type);
   aom_wb_write_bit(wb, cm->show_frame);
@@ -4671,18 +4636,12 @@
       }
     }
 #endif
-#if CONFIG_EXT_REFS
     cpi->refresh_frame_mask = get_refresh_mask(cpi);
-#endif  // CONFIG_EXT_REFS
 
     if (cm->intra_only) {
       write_bitdepth_colorspace_sampling(cm, wb);
 
-#if CONFIG_EXT_REFS
       aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
-#else
-      aom_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES);
-#endif  // CONFIG_EXT_REFS
       write_frame_size(cm, wb);
 
 #if CONFIG_ANS && ANS_MAX_SYMBOLS
@@ -4693,19 +4652,13 @@
     } else {
       MV_REFERENCE_FRAME ref_frame;
 
-#if CONFIG_EXT_REFS
       aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
-#else
-      aom_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES);
-#endif  // CONFIG_EXT_REFS
 
-#if CONFIG_EXT_REFS
       if (!cpi->refresh_frame_mask) {
         // NOTE: "cpi->refresh_frame_mask == 0" indicates that the coded frame
         //       will not be used as a reference
         cm->is_reference_frame = 0;
       }
-#endif  // CONFIG_EXT_REFS
 
       for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
         assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
@@ -4764,12 +4717,10 @@
     int arf_offset = AOMMIN(
         (MAX_GF_INTERVAL - 1),
         cpi->twopass.gf_group.arf_src_offset[cpi->twopass.gf_group.index]);
-#if CONFIG_EXT_REFS
     int brf_offset =
         cpi->twopass.gf_group.brf_src_offset[cpi->twopass.gf_group.index];
 
     arf_offset = AOMMIN((MAX_GF_INTERVAL - 1), arf_offset + brf_offset);
-#endif
     aom_wb_write_literal(wb, arf_offset, 4);
   }
 #endif
@@ -4872,7 +4823,6 @@
   aom_wb_write_literal(wb, cm->large_scale_tile, 1);
 #endif  // CONFIG_EXT_TILE
 
-#if CONFIG_EXT_REFS
   // NOTE: By default all coded frames to be used as a reference
   cm->is_reference_frame = 1;
 
@@ -4904,11 +4854,8 @@
 
     return;
   } else {
-#endif  // CONFIG_EXT_REFS
     aom_wb_write_bit(wb, 0);  // show_existing_frame
-#if CONFIG_EXT_REFS
   }
-#endif  // CONFIG_EXT_REFS
 
   cm->frame_type = cm->intra_only ? INTRA_ONLY_FRAME : cm->frame_type;
   aom_wb_write_literal(wb, cm->frame_type, 2);
@@ -4955,16 +4902,10 @@
       }
     }
 #endif
-#if CONFIG_EXT_REFS
     cpi->refresh_frame_mask = get_refresh_mask(cpi);
-#endif  // CONFIG_EXT_REFS
 
     if (cm->intra_only) {
-#if CONFIG_EXT_REFS
       aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
-#else
-      aom_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES);
-#endif  // CONFIG_EXT_REFS
       write_frame_size(cm, wb);
 
 #if CONFIG_ANS && ANS_MAX_SYMBOLS
@@ -4984,20 +4925,14 @@
     }
 #endif
 
-#if CONFIG_EXT_REFS
     cpi->refresh_frame_mask = get_refresh_mask(cpi);
     aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
-#else
-    aom_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES);
-#endif  // CONFIG_EXT_REFS
 
-#if CONFIG_EXT_REFS
     if (!cpi->refresh_frame_mask) {
       // NOTE: "cpi->refresh_frame_mask == 0" indicates that the coded frame
       //       will not be used as a reference
       cm->is_reference_frame = 0;
     }
-#endif  // CONFIG_EXT_REFS
 
     for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
       assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
@@ -5060,13 +4995,11 @@
     }
 #endif
 
-#if CONFIG_EXT_REFS
     if (!cpi->refresh_frame_mask) {
       // NOTE: "cpi->refresh_frame_mask == 0" indicates that the coded frame
       //       will not be used as a reference
       cm->is_reference_frame = 0;
     }
-#endif  // CONFIG_EXT_REFS
 
     for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
       assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
@@ -5117,12 +5050,10 @@
     int arf_offset = AOMMIN(
         (MAX_GF_INTERVAL - 1),
         cpi->twopass.gf_group.arf_src_offset[cpi->twopass.gf_group.index]);
-#if CONFIG_EXT_REFS
     int brf_offset =
         cpi->twopass.gf_group.brf_src_offset[cpi->twopass.gf_group.index];
 
     arf_offset = AOMMIN((MAX_GF_INTERVAL - 1), arf_offset + brf_offset);
-#endif
     aom_wb_write_literal(wb, arf_offset, 4);
   }
 #endif
@@ -5319,7 +5250,6 @@
 #endif  // CONFIG_EXT_COMP_REFS
 
       for (int i = 0; i < REF_CONTEXTS; i++) {
-#if CONFIG_EXT_REFS
         for (int j = 0; j < (FWD_REFS - 1); j++) {
           av1_cond_prob_diff_update(header_bc, &fc->comp_ref_prob[i][j],
                                     counts->comp_ref[i][j], probwt);
@@ -5328,12 +5258,6 @@
           av1_cond_prob_diff_update(header_bc, &fc->comp_bwdref_prob[i][j],
                                     counts->comp_bwdref[i][j], probwt);
         }
-#else
-        for (int j = 0; j < (COMP_REFS - 1); j++) {
-          av1_cond_prob_diff_update(header_bc, &fc->comp_ref_prob[i][j],
-                                    counts->comp_ref[i][j], probwt);
-        }
-#endif  // CONFIG_EXT_REFS
       }
     }
 #endif  // CONFIG_NEW_MULTISYMBOL
@@ -5898,12 +5822,10 @@
     // Write the uncompressed header
     write_uncompressed_header_frame(cpi, &wb);
 
-#if CONFIG_EXT_REFS
     if (cm->show_existing_frame) {
       *size = aom_wb_bytes_written(&wb);
       return;
     }
-#endif  // CONFIG_EXT_REFS
 
     // We do not know these in advance. Output placeholder bit.
     saved_wb = wb;
diff --git a/av1/encoder/bitstream.h b/av1/encoder/bitstream.h
index 76eb851..ec40a4e 100644
--- a/av1/encoder/bitstream.h
+++ b/av1/encoder/bitstream.h
@@ -30,14 +30,9 @@
 void av1_encode_token_init(void);
 
 static INLINE int av1_preserve_existing_gf(AV1_COMP *cpi) {
-#if CONFIG_EXT_REFS
   // Do not swap gf and arf indices for internal overlay frames
   return !cpi->multi_arf_allowed && cpi->rc.is_src_frame_alt_ref &&
          !cpi->rc.is_src_frame_ext_arf;
-#else
-  return !cpi->multi_arf_allowed && cpi->refresh_golden_frame &&
-         cpi->rc.is_src_frame_alt_ref;
-#endif  // CONFIG_EXT_REFS
 }
 
 void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 06261d1..eefeeab 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1523,9 +1523,7 @@
   if ((rd_cost->rate != INT_MAX) && (aq_mode == COMPLEXITY_AQ) &&
       (bsize >= BLOCK_16X16) &&
       (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame ||
-#if CONFIG_EXT_REFS
        cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
        (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref))) {
     av1_caq_select_segment(cpi, x, bsize, mi_row, mi_col, rd_cost->rate);
   }
@@ -1663,9 +1661,7 @@
       // the reference frame counts used to work out probabilities.
       if (inter_block) {
         const MV_REFERENCE_FRAME ref0 = mbmi->ref_frame[0];
-#if CONFIG_EXT_REFS
         const MV_REFERENCE_FRAME ref1 = mbmi->ref_frame[1];
-#endif  // CONFIG_EXT_REFS
 
         if (cm->reference_mode == REFERENCE_MODE_SELECT) {
           if (has_second_ref(mbmi))
@@ -1711,7 +1707,6 @@
             }
           } else {
 #endif  // CONFIG_EXT_COMP_REFS
-#if CONFIG_EXT_REFS
             const int bit = (ref0 == GOLDEN_FRAME || ref0 == LAST3_FRAME);
 
             counts->comp_ref[av1_get_pred_context_comp_ref_p(cm, xd)][0][bit]++;
@@ -1728,15 +1723,10 @@
             if (ref1 != ALTREF_FRAME)
               counts->comp_bwdref[av1_get_pred_context_comp_bwdref_p1(cm, xd)]
                                  [1][ref1 == ALTREF2_FRAME]++;
-#else   // !CONFIG_EXT_REFS
-          counts->comp_ref[av1_get_pred_context_comp_ref_p(cm, xd)][0]
-                          [ref0 == GOLDEN_FRAME]++;
-#endif  // CONFIG_EXT_REFS
 #if CONFIG_EXT_COMP_REFS
           }
 #endif  // CONFIG_EXT_COMP_REFS
         } else {
-#if CONFIG_EXT_REFS
           const int bit = (ref0 >= BWDREF_FRAME);
 
           counts->single_ref[av1_get_pred_context_single_ref_p1(xd)][0][bit]++;
@@ -1759,14 +1749,6 @@
                                 [ref0 != LAST3_FRAME]++;
             }
           }
-#else   // !CONFIG_EXT_REFS
-          counts->single_ref[av1_get_pred_context_single_ref_p1(xd)][0]
-                            [ref0 != LAST_FRAME]++;
-          if (ref0 != LAST_FRAME) {
-            counts->single_ref[av1_get_pred_context_single_ref_p2(xd)][1]
-                              [ref0 != GOLDEN_FRAME]++;
-          }
-#endif  // CONFIG_EXT_REFS
         }
 
 #if CONFIG_COMPOUND_SINGLEREF
@@ -4817,10 +4799,8 @@
     return 0;
   } else {
     return (!!(ref_flags & AOM_GOLD_FLAG) + !!(ref_flags & AOM_LAST_FLAG) +
-#if CONFIG_EXT_REFS
             !!(ref_flags & AOM_LAST2_FLAG) + !!(ref_flags & AOM_LAST3_FLAG) +
             !!(ref_flags & AOM_BWD_FLAG) + !!(ref_flags & AOM_ALT2_FLAG) +
-#endif  // CONFIG_EXT_REFS
             !!(ref_flags & AOM_ALT_FLAG)) >= 2;
   }
 }
@@ -4843,18 +4823,11 @@
 
 static MV_REFERENCE_FRAME get_frame_type(const AV1_COMP *cpi) {
   if (frame_is_intra_only(&cpi->common)) return INTRA_FRAME;
-#if CONFIG_EXT_REFS
   // We will not update the golden frame with an internal overlay frame
   else if ((cpi->rc.is_src_frame_alt_ref && cpi->refresh_golden_frame) ||
            cpi->rc.is_src_frame_ext_arf)
-#else
-  else if (cpi->rc.is_src_frame_alt_ref && cpi->refresh_golden_frame)
-#endif  // CONFIG_EXT_REFS
     return ALTREF_FRAME;
-  else if (cpi->refresh_golden_frame ||
-#if CONFIG_EXT_REFS
-           cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
+  else if (cpi->refresh_golden_frame || cpi->refresh_alt2_ref_frame ||
            cpi->refresh_alt_ref_frame)
     return GOLDEN_FRAME;
   else
@@ -5170,11 +5143,7 @@
   switch (sf->gm_search_type) {
     case GM_FULL_SEARCH: return 1;
     case GM_REDUCED_REF_SEARCH:
-#if CONFIG_EXT_REFS
       return !(frame == LAST2_FRAME || frame == LAST3_FRAME);
-#else
-      return (num_refs_using_gm < 2);
-#endif  // CONFIG_EXT_REFS
     case GM_DISABLE_SEARCH: return 0;
     default: assert(0);
   }
@@ -5220,9 +5189,7 @@
   MACROBLOCKD *const xd = &x->e_mbd;
   RD_COUNTS *const rdc = &cpi->td.rd_counts;
   int i;
-#if CONFIG_TEMPMV_SIGNALING || CONFIG_EXT_REFS
   const int last_fb_buf_idx = get_ref_frame_buf_idx(cpi, LAST_FRAME);
-#endif  // CONFIG_TEMPMV_SIGNALING || CONFIG_EXT_REFS
 
 #if CONFIG_ADAPT_SCAN
   av1_deliver_eob_threshold(cm, xd);
@@ -5488,7 +5455,6 @@
   av1_initialize_me_consts(cpi, x, cm->base_qindex);
   init_encode_frame_mb_context(cpi);
 
-#if CONFIG_EXT_REFS || CONFIG_TEMPMV_SIGNALING
   // NOTE(zoeliu): As cm->prev_frame can take neither a frame of
   //               show_exisiting_frame=1, nor can it take a frame not used as
   //               a reference, it is probable that by the time it is being
@@ -5505,7 +5471,6 @@
                          ? &cm->buffer_pool->frame_bufs[last_fb_buf_idx]
                          : NULL;
   }
-#endif  // CONFIG_EXT_REFS || CONFIG_TEMPMV_SIGNALING
 
 #if CONFIG_TEMPMV_SIGNALING
   cm->use_prev_frame_mvs &= frame_can_use_prev_frame_mvs(cm);
@@ -5609,11 +5574,9 @@
     int arf_offset = AOMMIN(
         (MAX_GF_INTERVAL - 1),
         cpi->twopass.gf_group.arf_src_offset[cpi->twopass.gf_group.index]);
-#if CONFIG_EXT_REFS
     int brf_offset =
         cpi->twopass.gf_group.brf_src_offset[cpi->twopass.gf_group.index];
     arf_offset = AOMMIN((MAX_GF_INTERVAL - 1), arf_offset + brf_offset);
-#endif  // CONFIG_EXT_REFS
     cm->frame_offset = cm->current_video_frame + arf_offset;
   } else {
     cm->frame_offset = cm->current_video_frame;
@@ -5640,7 +5603,6 @@
     } else {
 #endif  // !CONFIG_ONE_SIDED_COMPOUND
       cpi->allow_comp_inter_inter = 1;
-#if CONFIG_EXT_REFS
       cm->comp_fwd_ref[0] = LAST_FRAME;
       cm->comp_fwd_ref[1] = LAST2_FRAME;
       cm->comp_fwd_ref[2] = LAST3_FRAME;
@@ -5648,11 +5610,6 @@
       cm->comp_bwd_ref[0] = BWDREF_FRAME;
       cm->comp_bwd_ref[1] = ALTREF2_FRAME;
       cm->comp_bwd_ref[2] = ALTREF_FRAME;
-#else                           // !CONFIG_EXT_REFS
-    cm->comp_fixed_ref = ALTREF_FRAME;
-    cm->comp_var_ref[0] = LAST_FRAME;
-    cm->comp_var_ref[1] = GOLDEN_FRAME;
-#endif                          // CONFIG_EXT_REFS
 #if !CONFIG_ONE_SIDED_COMPOUND  // Normative in encoder
     }
 #endif  // !CONFIG_ONE_SIDED_COMPOUND
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index ecb41ff..1208e4f 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -341,23 +341,17 @@
 // Just use frame context from first signaled reference frame.
 // This will always be LAST_FRAME for now.
 #else
-#if CONFIG_EXT_REFS
     const GF_GROUP *gf_group = &cpi->twopass.gf_group;
     if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE)
       cm->frame_context_idx = EXT_ARF_FRAME;
     else if (cpi->refresh_alt_ref_frame)
       cm->frame_context_idx = ARF_FRAME;
-#else   // !CONFIG_EXT_REFS
-    if (cpi->refresh_alt_ref_frame) cm->frame_context_idx = ARF_FRAME;
-#endif  // CONFIG_EXT_REFS
     else if (cpi->rc.is_src_frame_alt_ref)
       cm->frame_context_idx = OVERLAY_FRAME;
     else if (cpi->refresh_golden_frame)
       cm->frame_context_idx = GLD_FRAME;
-#if CONFIG_EXT_REFS
     else if (cpi->refresh_bwd_ref_frame)
       cm->frame_context_idx = BRF_FRAME;
-#endif  // CONFIG_EXT_REFS
     else
       cm->frame_context_idx = REGULAR_FRAME;
 #endif  // CONFIG_NO_FRAME_CONTEXT_SIGNALING
@@ -384,7 +378,6 @@
 #endif  // CONFIG_NO_FRAME_CONTEXT_SIGNALING
     av1_zero(cpi->interp_filter_selected[0]);
   }
-#if CONFIG_EXT_REFS
 #if CONFIG_ONE_SIDED_COMPOUND && \
     !CONFIG_EXT_COMP_REFS  // No change to bitstream
   if (cpi->sf.recode_loop == DISALLOW_RECODE) {
@@ -392,7 +385,6 @@
     cpi->rc.is_bipred_frame = 1;
   }
 #endif  // CONFIG_ONE_SIDED_COMPOUND && !CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 #if CONFIG_NO_FRAME_CONTEXT_SIGNALING
   if (frame_is_intra_only(cm) || cm->error_resilient_mode ||
       cm->frame_refs[0].idx < 0) {
@@ -1093,7 +1085,6 @@
 }
 
 static void init_buffer_indices(AV1_COMP *cpi) {
-#if CONFIG_EXT_REFS
   int fb_idx;
   for (fb_idx = 0; fb_idx < LAST_REF_FRAMES; ++fb_idx)
     cpi->lst_fb_idxes[fb_idx] = fb_idx;
@@ -1104,11 +1095,6 @@
   cpi->ext_fb_idx = LAST_REF_FRAMES + 4;
   for (fb_idx = 0; fb_idx < MAX_EXT_ARFS + 1; ++fb_idx)
     cpi->arf_map[fb_idx] = LAST_REF_FRAMES + 2 + fb_idx;
-#else   // !CONFIG_EXT_REFS
-  cpi->lst_fb_idx = 0;
-  cpi->gld_fb_idx = 1;
-  cpi->alt_fb_idx = 2;
-#endif  // CONFIG_EXT_REFS
 #if CONFIG_AMVR
   cpi->rate_index = 0;
   cpi->rate_size = 0;
@@ -2525,10 +2511,8 @@
 
   cpi->refresh_last_frame = 1;
   cpi->refresh_golden_frame = 0;
-#if CONFIG_EXT_REFS
   cpi->refresh_bwd_ref_frame = 0;
   cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
 
   cm->refresh_frame_context =
       (oxcf->error_resilient_mode || oxcf->frame_parallel_decoding_mode)
@@ -2590,11 +2574,9 @@
   cpi->alt_ref_source = NULL;
   rc->is_src_frame_alt_ref = 0;
 
-#if CONFIG_EXT_REFS
   rc->is_bwd_ref_frame = 0;
   rc->is_last_bipred_frame = 0;
   rc->is_bipred_frame = 0;
-#endif  // CONFIG_EXT_REFS
 
 #if 0
   // Experimental RD Code
@@ -3356,7 +3338,7 @@
 }
 #endif
 
-#if CONFIG_EXT_REFS && !CONFIG_XIPHRC
+#if !CONFIG_XIPHRC
 #if USE_GF16_MULTI_LAYER
 static void check_show_existing_frame_gf16(AV1_COMP *cpi) {
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
@@ -3433,7 +3415,7 @@
   }
   cpi->rc.is_src_frame_ext_arf = 0;
 }
-#endif  // CONFIG_EXT_REFS && !CONFIG_XIPHRC
+#endif  // !CONFIG_XIPHRC
 
 #ifdef OUTPUT_YUV_REC
 void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) {
@@ -3597,7 +3579,6 @@
 }
 #endif  // DUMP_REF_FRAME_IMAGES == 1
 
-#if CONFIG_EXT_REFS
 // This function is used to shift the virtual indices of last reference frames
 // as follows:
 // LAST_FRAME -> LAST2_FRAME -> LAST3_FRAME
@@ -3616,7 +3597,6 @@
     }
   }
 }
-#endif  // CONFIG_EXT_REFS
 
 #if CONFIG_VAR_REFS
 static void enc_check_valid_ref_frames(AV1_COMP *const cpi) {
@@ -3654,7 +3634,6 @@
 }
 #endif  // CONFIG_VAR_REFS
 
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
 static void update_reference_frames_gf16(AV1_COMP *cpi) {
   AV1_COMMON *const cm = &cpi->common;
@@ -3698,7 +3677,6 @@
 #endif  // DUMP_REF_FRAME_IMAGES
 }
 #endif  // USE_GF16_MULTI_LAYER
-#endif  // CONFIG_EXT_REFS
 
 static void update_reference_frames(AV1_COMP *cpi) {
   AV1_COMMON *const cm = &cpi->common;
@@ -3707,14 +3685,12 @@
   //       for the purpose to verify no mismatch between encoder and decoder.
   if (cm->show_frame) cpi->last_show_frame_buf_idx = cm->new_fb_idx;
 
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
   if (cpi->rc.baseline_gf_interval == 16) {
     update_reference_frames_gf16(cpi);
     return;
   }
 #endif  // USE_GF16_MULTI_LAYER
-#endif  // CONFIG_EXT_REFS
 
   BufferPool *const pool = cm->buffer_pool;
   // At this point the new frame has been encoded.
@@ -3722,12 +3698,10 @@
   if (cm->frame_type == KEY_FRAME) {
     ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx],
                cm->new_fb_idx);
-#if CONFIG_EXT_REFS
     ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->bwd_fb_idx],
                cm->new_fb_idx);
     ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->alt2_fb_idx],
                cm->new_fb_idx);
-#endif  // CONFIG_EXT_REFS
     ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->alt_fb_idx],
                cm->new_fb_idx);
   } else if (av1_preserve_existing_gf(cpi)) {
@@ -3747,13 +3721,10 @@
     cpi->alt_fb_idx = cpi->gld_fb_idx;
     cpi->gld_fb_idx = tmp;
 
-#if CONFIG_EXT_REFS
     // We need to modify the mapping accordingly
     cpi->arf_map[0] = cpi->alt_fb_idx;
-#endif  // CONFIG_EXT_REFS
-// TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to
-// cpi->interp_filter_selected[GOLDEN_FRAME]?
-#if CONFIG_EXT_REFS
+    // TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to
+    // cpi->interp_filter_selected[GOLDEN_FRAME]?
   } else if (cpi->rc.is_src_frame_ext_arf && cm->show_existing_frame) {
     // Deal with the special case for showing existing internal ALTREF_FRAME
     // Refresh the LAST_FRAME with the ALTREF_FRAME and retire the LAST3_FRAME
@@ -3773,18 +3744,11 @@
     memcpy(cpi->interp_filter_selected[LAST_FRAME],
            cpi->interp_filter_selected[ALTREF2_FRAME],
            sizeof(cpi->interp_filter_selected[ALTREF2_FRAME]));
-#endif     // CONFIG_EXT_REFS
   } else { /* For non key/golden frames */
     // === ALTREF_FRAME ===
     if (cpi->refresh_alt_ref_frame) {
       int arf_idx = cpi->alt_fb_idx;
       int which_arf = 0;
-#if !CONFIG_EXT_REFS
-      if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) {
-        const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
-        arf_idx = gf_group->arf_update_idx[gf_group->index];
-      }
-#endif  // !CONFIG_EXT_REFS
       ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[arf_idx], cm->new_fb_idx);
 
       memcpy(cpi->interp_filter_selected[ALTREF_FRAME + which_arf],
@@ -3797,15 +3761,11 @@
       ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx],
                  cm->new_fb_idx);
 
-#if !CONFIG_EXT_REFS
-      if (!cpi->rc.is_src_frame_alt_ref)
-#endif  // !CONFIG_EXT_REFS
-        memcpy(cpi->interp_filter_selected[GOLDEN_FRAME],
-               cpi->interp_filter_selected[0],
-               sizeof(cpi->interp_filter_selected[0]));
+      memcpy(cpi->interp_filter_selected[GOLDEN_FRAME],
+             cpi->interp_filter_selected[0],
+             sizeof(cpi->interp_filter_selected[0]));
     }
 
-#if CONFIG_EXT_REFS
     // === BWDREF_FRAME ===
     if (cpi->refresh_bwd_ref_frame) {
       ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->bwd_fb_idx],
@@ -3825,11 +3785,9 @@
              cpi->interp_filter_selected[0],
              sizeof(cpi->interp_filter_selected[0]));
     }
-#endif  // CONFIG_EXT_REFS
   }
 
   if (cpi->refresh_last_frame) {
-#if CONFIG_EXT_REFS
     // NOTE(zoeliu): We have two layers of mapping (1) from the per-frame
     // reference to the reference frame buffer virtual index; and then (2) from
     // the virtual index to the reference frame buffer physical index:
@@ -3905,15 +3863,6 @@
                sizeof(cpi->interp_filter_selected[BWDREF_FRAME]));
       }
     }
-#else   // !CONFIG_EXT_REFS
-    ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->lst_fb_idx],
-               cm->new_fb_idx);
-    if (!cpi->rc.is_src_frame_alt_ref) {
-      memcpy(cpi->interp_filter_selected[LAST_FRAME],
-             cpi->interp_filter_selected[0],
-             sizeof(cpi->interp_filter_selected[0]));
-    }
-#endif  // CONFIG_EXT_REFS
   }
 
 #if DUMP_REF_FRAME_IMAGES == 1
@@ -3934,17 +3883,8 @@
   AV1_COMMON *cm = &cpi->common;
   MV_REFERENCE_FRAME ref_frame;
   const AOM_REFFRAME ref_mask[INTER_REFS_PER_FRAME] = {
-    AOM_LAST_FLAG,
-#if CONFIG_EXT_REFS
-    AOM_LAST2_FLAG,
-    AOM_LAST3_FLAG,
-#endif  // CONFIG_EXT_REFS
-    AOM_GOLD_FLAG,
-#if CONFIG_EXT_REFS
-    AOM_BWD_FLAG,
-    AOM_ALT2_FLAG,
-#endif  // CONFIG_EXT_REFS
-    AOM_ALT_FLAG
+    AOM_LAST_FLAG, AOM_LAST2_FLAG, AOM_LAST3_FLAG, AOM_GOLD_FLAG,
+    AOM_BWD_FLAG,  AOM_ALT2_FLAG,  AOM_ALT_FLAG
   };
 
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
@@ -4029,16 +3969,11 @@
     // if reference will be updated, or if scaled reference has same resolution.
     int refresh[INTER_REFS_PER_FRAME];
     refresh[0] = (cpi->refresh_last_frame) ? 1 : 0;
-#if CONFIG_EXT_REFS
     refresh[1] = refresh[2] = 0;
     refresh[3] = (cpi->refresh_golden_frame) ? 1 : 0;
     refresh[4] = (cpi->refresh_bwd_ref_frame) ? 1 : 0;
     refresh[5] = (cpi->refresh_alt2_ref_frame) ? 1 : 0;
     refresh[6] = (cpi->refresh_alt_ref_frame) ? 1 : 0;
-#else   // !CONFIG_EXT_REFS
-    refresh[1] = (cpi->refresh_golden_frame) ? 1 : 0;
-    refresh[2] = (cpi->refresh_alt_ref_frame) ? 1 : 0;
-#endif  // CONFIG_EXT_REFS
     for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
       const int idx = cpi->scaled_ref_idx[i - 1];
       RefCntBuffer *const buf =
@@ -5099,7 +5034,6 @@
 static int get_ref_frame_flags(const AV1_COMP *cpi) {
   const int *const map = cpi->common.ref_frame_map;
 
-#if CONFIG_EXT_REFS
   const int last2_is_last =
       map[cpi->lst_fb_idxes[1]] == map[cpi->lst_fb_idxes[0]];
   const int last3_is_last =
@@ -5137,11 +5071,6 @@
   const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
   const int bwd_is_alt = map[cpi->bwd_fb_idx] == map[cpi->alt_fb_idx];
   const int alt2_is_alt = map[cpi->alt2_fb_idx] == map[cpi->alt_fb_idx];
-#else   // !CONFIG_EXT_REFS
-  const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idx];
-  const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
-  const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idx];
-#endif  // CONFIG_EXT_REFS
 
   int flags = AOM_REFFRAME_ALL;
 
@@ -5151,7 +5080,6 @@
 
   if (alt_is_last) flags &= ~AOM_ALT_FLAG;
 
-#if CONFIG_EXT_REFS
   if (last2_is_last || last2_is_alt) flags &= ~AOM_LAST2_FLAG;
 
   if (last3_is_last || last3_is_last2 || last3_is_alt) flags &= ~AOM_LAST3_FLAG;
@@ -5173,7 +5101,6 @@
        alt2_is_bwd || alt2_is_alt) &&
       (flags & AOM_ALT2_FLAG))
     flags &= ~AOM_ALT2_FLAG;
-#endif  // CONFIG_EXT_REFS
 
   return flags;
 }
@@ -5199,30 +5126,16 @@
 static void set_arf_sign_bias(AV1_COMP *cpi) {
   AV1_COMMON *const cm = &cpi->common;
   int arf_sign_bias;
-#if CONFIG_EXT_REFS
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
   // The arf_sign_bias will be one for internal ARFs'
   arf_sign_bias = cpi->rc.source_alt_ref_active &&
                   (!cpi->refresh_alt_ref_frame ||
                    gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE);
-#else   // !CONFIG_EXT_REFS
-  if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) {
-    const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
-    arf_sign_bias = cpi->rc.source_alt_ref_active &&
-                    (!cpi->refresh_alt_ref_frame ||
-                     (gf_group->rf_level[gf_group->index] == GF_ARF_LOW));
-  } else {
-    arf_sign_bias =
-        (cpi->rc.source_alt_ref_active && !cpi->refresh_alt_ref_frame);
-  }
-#endif  // CONFIG_EXT_REFS
 
   cm->ref_frame_sign_bias[ALTREF_FRAME] = arf_sign_bias;
-#if CONFIG_EXT_REFS
   cm->ref_frame_sign_bias[BWDREF_FRAME] = cm->ref_frame_sign_bias[ALTREF_FRAME];
   cm->ref_frame_sign_bias[ALTREF2_FRAME] =
       cm->ref_frame_sign_bias[ALTREF_FRAME];
-#endif  // CONFIG_EXT_REFS
 }
 #endif  // !CONFIG_FRAME_SIGN_BIAS
 
@@ -5233,12 +5146,8 @@
   int mask = 0;
   int arf_idx = ALTREF_FRAME;
 
-#if CONFIG_EXT_REFS
   if (cpi->common.last_frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame ||
       cpi->refresh_alt2_ref_frame)
-#else   // !CONFIG_EXT_REFS
-  if (cpi->common.last_frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame)
-#endif  // CONFIG_EXT_REFS
     return mask;
 
   for (ref = LAST_FRAME; ref <= ALTREF_FRAME; ++ref)
@@ -5248,25 +5157,21 @@
   for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter) {
     if ((ref_total[LAST_FRAME] &&
          cpi->interp_filter_selected[LAST_FRAME][ifilter] == 0) &&
-#if CONFIG_EXT_REFS
         (ref_total[LAST2_FRAME] == 0 ||
          cpi->interp_filter_selected[LAST2_FRAME][ifilter] * 50 <
              ref_total[LAST2_FRAME]) &&
         (ref_total[LAST3_FRAME] == 0 ||
          cpi->interp_filter_selected[LAST3_FRAME][ifilter] * 50 <
              ref_total[LAST3_FRAME]) &&
-#endif  // CONFIG_EXT_REFS
         (ref_total[GOLDEN_FRAME] == 0 ||
          cpi->interp_filter_selected[GOLDEN_FRAME][ifilter] * 50 <
              ref_total[GOLDEN_FRAME]) &&
-#if CONFIG_EXT_REFS
         (ref_total[BWDREF_FRAME] == 0 ||
          cpi->interp_filter_selected[BWDREF_FRAME][ifilter] * 50 <
              ref_total[BWDREF_FRAME]) &&
         (ref_total[ALTREF2_FRAME] == 0 ||
          cpi->interp_filter_selected[ALTREF2_FRAME][ifilter] * 50 <
              ref_total[ALTREF2_FRAME]) &&
-#endif  // CONFIG_EXT_REFS
         (ref_total[ALTREF_FRAME] == 0 ||
          cpi->interp_filter_selected[arf_idx][ifilter] * 50 <
              ref_total[ALTREF_FRAME]))
@@ -5390,7 +5295,6 @@
       !cpi->oxcf.disable_tempmv && !cm->cur_frame->intra_only;
 #endif
 
-#if CONFIG_EXT_REFS
   // NOTE:
   // (1) Move the setup of the ref_frame_flags upfront as it would be
   //     determined by the current frame properties;
@@ -5474,7 +5378,6 @@
     aom_free(cdf_ptrs);
     return;
   }
-#endif  // CONFIG_EXT_REFS
 
   // Set default state for segment based loop filter update flags.
   cm->lf.mode_ref_delta_update = 0;
@@ -5634,10 +5537,8 @@
   cm->frame_to_show->render_width = cm->render_width;
   cm->frame_to_show->render_height = cm->render_height;
 
-#if CONFIG_EXT_REFS
-// TODO(zoeliu): For non-ref frames, loop filtering may need to be turned
-// off.
-#endif  // CONFIG_EXT_REFS
+  // TODO(zoeliu): For non-ref frames, loop filtering may need to be turned
+  // off.
 
   // Pick the loop filter level for the frame.
   loopfilter_frame(cpi, cm);
@@ -5729,16 +5630,10 @@
   else
     cpi->frame_flags &= ~FRAMEFLAGS_ALTREF;
 
-#if CONFIG_EXT_REFS
   if (cpi->refresh_bwd_ref_frame == 1)
     cpi->frame_flags |= FRAMEFLAGS_BWDREF;
   else
     cpi->frame_flags &= ~FRAMEFLAGS_BWDREF;
-#endif  // CONFIG_EXT_REFS
-
-#if !CONFIG_EXT_REFS
-  cpi->ref_frame_flags = get_ref_frame_flags(cpi);
-#endif  // !CONFIG_EXT_REFS
 
   cm->last_frame_type = cm->frame_type;
 
@@ -5777,20 +5672,16 @@
   cm->lf.mode_ref_delta_update = 0;
 
   if (cm->show_frame) {
-#if CONFIG_EXT_REFS
-// TODO(zoeliu): We may only swamp mi and prev_mi for those frames that are
-// being used as reference.
-#endif  // CONFIG_EXT_REFS
+    // TODO(zoeliu): We may only swamp mi and prev_mi for those frames that are
+    // being used as reference.
     swap_mi_and_prev_mi(cm);
     // Don't increment frame counters if this was an altref buffer
     // update not a real frame
     ++cm->current_video_frame;
   }
 
-#if CONFIG_EXT_REFS
   // NOTE: Shall not refer to any frame not used as reference.
   if (cm->is_reference_frame) {
-#endif  // CONFIG_EXT_REFS
     cm->prev_frame = cm->cur_frame;
     // keep track of the last coded dimensions
     cm->last_width = cm->width;
@@ -5798,9 +5689,7 @@
 
     // reset to normal state now that we are done.
     cm->last_show_frame = cm->show_frame;
-#if CONFIG_EXT_REFS
   }
-#endif  // CONFIG_EXT_REFS
 
   aom_free(tile_ctxs);
   aom_free(cdf_ptrs);
@@ -5848,7 +5737,6 @@
                         unsigned int *frame_flags) {
   encode_frame_to_data_rate(cpi, size, dest, 0, frame_flags);
 
-#if CONFIG_EXT_REFS
   // Do not do post-encoding update for those frames that do not have a spot in
   // a gf group, but note that an OVERLAY frame always has a spot in a gf group,
   // even when show_existing_frame is used.
@@ -5856,9 +5744,6 @@
     av1_twopass_postencode_update(cpi);
   }
   check_show_existing_frame(cpi);
-#else
-  av1_twopass_postencode_update(cpi);
-#endif  // CONFIG_EXT_REFS
 }
 #endif
 
@@ -5911,13 +5796,10 @@
   const AV1_COMMON *cm = &cpi->common;
 
   return cm->frame_type == KEY_FRAME || cpi->refresh_last_frame ||
-         cpi->refresh_golden_frame ||
-#if CONFIG_EXT_REFS
-         cpi->refresh_bwd_ref_frame || cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
-         cpi->refresh_alt_ref_frame || !cm->error_resilient_mode ||
-         cm->lf.mode_ref_delta_update || cm->seg.update_map ||
-         cm->seg.update_data;
+         cpi->refresh_golden_frame || cpi->refresh_bwd_ref_frame ||
+         cpi->refresh_alt2_ref_frame || cpi->refresh_alt_ref_frame ||
+         !cm->error_resilient_mode || cm->lf.mode_ref_delta_update ||
+         cm->seg.update_map || cm->seg.update_data;
 }
 
 static void adjust_frame_rate(AV1_COMP *cpi,
@@ -5977,7 +5859,6 @@
   return arf_src_index;
 }
 
-#if CONFIG_EXT_REFS
 static int get_brf_src_index(AV1_COMP *cpi) {
   int brf_src_index = 0;
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
@@ -6011,7 +5892,6 @@
   }
   return arf2_src_index;
 }
-#endif  // CONFIG_EXT_REFS
 
 static void check_src_altref(AV1_COMP *cpi,
                              const struct lookahead_entry *source) {
@@ -6023,14 +5903,10 @@
   if (cpi->oxcf.pass == 2) {
     const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
     rc->is_src_frame_alt_ref =
-#if CONFIG_EXT_REFS
         (gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE) ||
-#endif  // CONFIG_EXT_REFS
         (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE);
-#if CONFIG_EXT_REFS
     rc->is_src_frame_ext_arf =
         gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE;
-#endif  // CONFIG_EXT_REFS
   } else {
     rc->is_src_frame_alt_ref =
         cpi->alt_ref_source && (source == cpi->alt_ref_source);
@@ -6040,20 +5916,16 @@
     // Current frame is an ARF overlay frame.
     cpi->alt_ref_source = NULL;
 
-#if CONFIG_EXT_REFS
     if (rc->is_src_frame_ext_arf && !cpi->common.show_existing_frame) {
       // For INTNL_OVERLAY, when show_existing_frame == 0, they do need to
       // refresh the LAST_FRAME, i.e. LAST3 gets retired, LAST2 becomes LAST3,
       // LAST becomes LAST2, and INTNL_OVERLAY becomes LAST.
       cpi->refresh_last_frame = 1;
     } else {
-#endif  // CONFIG_EXT_REFS
       // Don't refresh the last buffer for an ARF overlay frame. It will
       // become the GF so preserve last as an alternative prediction option.
       cpi->refresh_last_frame = 0;
-#if CONFIG_EXT_REFS
     }
-#endif  // CONFIG_EXT_REFS
   }
 }
 
@@ -6305,9 +6177,7 @@
   struct lookahead_entry *last_source = NULL;
   struct lookahead_entry *source = NULL;
   int arf_src_index;
-#if CONFIG_EXT_REFS
   int brf_src_index;
-#endif  // CONFIG_EXT_REFS
   int i;
 
 #if CONFIG_XIPHRC
@@ -6347,13 +6217,11 @@
 
   cpi->refresh_last_frame = 1;
   cpi->refresh_golden_frame = 0;
-#if CONFIG_EXT_REFS
   cpi->refresh_bwd_ref_frame = 0;
   cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
   cpi->refresh_alt_ref_frame = 0;
 
-#if CONFIG_EXT_REFS && !CONFIG_XIPHRC
+#if !CONFIG_XIPHRC
   if (oxcf->pass == 2 && cm->show_existing_frame) {
     // Manage the source buffer and flush out the source frame that has been
     // coded already; Also get prepared for PSNR calculation if needed.
@@ -6402,7 +6270,7 @@
     cm->show_existing_frame = 0;
     return 0;
   }
-#endif  // CONFIG_EXT_REFS && !CONFIG_XIPHRC
+#endif  // !CONFIG_XIPHRC
 
   // Should we encode an arf frame.
   arf_src_index = get_arf_src_index(cpi);
@@ -6447,16 +6315,13 @@
       cpi->refresh_alt_ref_frame = 1;
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 0;
-#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
       cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
       rc->is_src_frame_alt_ref = 0;
     }
     rc->source_alt_ref_pending = 0;
   }
 
-#if CONFIG_EXT_REFS
   // Should we encode an arf2 frame.
   arf_src_index = get_arf2_src_index(cpi);
   if (arf_src_index) {
@@ -6520,7 +6385,6 @@
       rc->is_bwd_ref_frame = 1;
     }
   }
-#endif  // CONFIG_EXT_REFS
 
   if (!source) {
     // Get last frame source.
@@ -6586,16 +6450,6 @@
 #if CONFIG_HIGHBITDEPTH && CONFIG_GLOBAL_MOTION
   cm->cur_frame->buf.buf_8bit_valid = 0;
 #endif
-#if !CONFIG_EXT_REFS
-  if (cpi->multi_arf_allowed) {
-    if (cm->frame_type == KEY_FRAME) {
-      init_buffer_indices(cpi);
-    } else if (oxcf->pass == 2) {
-      const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
-      cpi->alt_fb_idx = gf_group->arf_ref_idx[gf_group->index];
-    }
-  }
-#endif  // !CONFIG_EXT_REFS
 
   // Start with a 0 size frame.
   *size = 0;
@@ -6789,10 +6643,8 @@
 
     if (flags & AOM_EFLAG_NO_REF_LAST) {
       ref ^= AOM_LAST_FLAG;
-#if CONFIG_EXT_REFS
       ref ^= AOM_LAST2_FLAG;
       ref ^= AOM_LAST3_FLAG;
-#endif  // CONFIG_EXT_REFS
     }
 
     if (flags & AOM_EFLAG_NO_REF_GF) ref ^= AOM_GOLD_FLAG;
@@ -6809,10 +6661,8 @@
 
     if (flags & AOM_EFLAG_NO_UPD_LAST) {
       upd ^= AOM_LAST_FLAG;
-#if CONFIG_EXT_REFS
       upd ^= AOM_LAST2_FLAG;
       upd ^= AOM_LAST3_FLAG;
-#endif  // CONFIG_EXT_REFS
     }
 
     if (flags & AOM_EFLAG_NO_UPD_GF) upd ^= AOM_GOLD_FLAG;
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 426867e..f95cd88 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -76,12 +76,10 @@
   OVERLAY_FRAME = 2,
   // golden frame
   GLD_FRAME = 3,
-#if CONFIG_EXT_REFS
   // backward reference frame
   BRF_FRAME = 4,
   // extra alternate reference frame
   EXT_ARF_FRAME = 5
-#endif
 } FRAME_CONTEXT_INDEX;
 #endif
 
@@ -101,13 +99,9 @@
 typedef enum {
   FRAMEFLAGS_KEY = 1 << 0,
   FRAMEFLAGS_GOLDEN = 1 << 1,
-#if CONFIG_EXT_REFS
   FRAMEFLAGS_BWDREF = 1 << 2,
   // TODO(zoeliu): To determine whether a frame flag is needed for ALTREF2_FRAME
   FRAMEFLAGS_ALTREF = 1 << 3,
-#else   // !CONFIG_EXT_REFS
-  FRAMEFLAGS_ALTREF = 1 << 2,
-#endif  // CONFIG_EXT_REFS
 } FRAMETYPE_FLAGS;
 
 typedef enum {
@@ -241,9 +235,7 @@
   // ----------------------------------------------------------------
 
   int enable_auto_arf;
-#if CONFIG_EXT_REFS
   int enable_auto_brf;  // (b)ackward (r)ef (f)rame
-#endif                  // CONFIG_EXT_REFS
 
   /* Bitfield defining the error resiliency features to enable.
    * Can provide decodable frames after losses in previous
@@ -422,30 +414,20 @@
 #endif
 
   int scaled_ref_idx[TOTAL_REFS_PER_FRAME];
-#if CONFIG_EXT_REFS
   int lst_fb_idxes[LAST_REF_FRAMES];
-#else
-  int lst_fb_idx;
-#endif  // CONFIG_EXT_REFS
   int gld_fb_idx;
-#if CONFIG_EXT_REFS
   int bwd_fb_idx;   // BWDREF_FRAME
   int alt2_fb_idx;  // ALTREF2_FRAME
-#endif              // CONFIG_EXT_REFS
   int alt_fb_idx;
-#if CONFIG_EXT_REFS
   int ext_fb_idx;      // extra ref frame buffer index
   int refresh_fb_idx;  // ref frame buffer index to refresh
-#endif                 // CONFIG_EXT_REFS
 
   int last_show_frame_buf_idx;  // last show frame buffer index
 
   int refresh_last_frame;
   int refresh_golden_frame;
-#if CONFIG_EXT_REFS
   int refresh_bwd_ref_frame;
   int refresh_alt2_ref_frame;
-#endif  // CONFIG_EXT_REFS
   int refresh_alt_ref_frame;
 
   int ext_refresh_frame_flags_pending;
@@ -607,7 +589,6 @@
 #if CONFIG_ANS
   struct BufAnsCoder buf_ans;
 #endif
-#if CONFIG_EXT_REFS
   int refresh_frame_mask;
   int existing_fb_idx_to_show;
   int is_arf_filter_off[MAX_EXT_ARFS + 1];
@@ -615,7 +596,6 @@
   int arf_map[MAX_EXT_ARFS + 1];
   int arf_pos_in_gf[MAX_EXT_ARFS + 1];
   int arf_pos_for_ovrly[MAX_EXT_ARFS + 1];
-#endif  // CONFIG_EXT_REFS
 #if CONFIG_GLOBAL_MOTION
   int global_motion_search_done;
 #endif
@@ -623,10 +603,8 @@
   tran_low_t *tcoeff_buf[MAX_MB_PLANE];
 #endif
 
-#if CONFIG_EXT_REFS
   int extra_arf_allowed;
   int bwd_ref_allowed;
-#endif  // CONFIG_EXT_REFS
 
 #if CONFIG_BGSPRITE
   int bgsprite_allowed;
@@ -681,20 +659,14 @@
 
 static INLINE int get_ref_frame_map_idx(const AV1_COMP *cpi,
                                         MV_REFERENCE_FRAME ref_frame) {
-#if CONFIG_EXT_REFS
   if (ref_frame >= LAST_FRAME && ref_frame <= LAST3_FRAME)
     return cpi->lst_fb_idxes[ref_frame - 1];
-#else
-  if (ref_frame == LAST_FRAME) return cpi->lst_fb_idx;
-#endif  // CONFIG_EXT_REFS
   else if (ref_frame == GOLDEN_FRAME)
     return cpi->gld_fb_idx;
-#if CONFIG_EXT_REFS
   else if (ref_frame == BWDREF_FRAME)
     return cpi->bwd_fb_idx;
   else if (ref_frame == ALTREF2_FRAME)
     return cpi->alt2_fb_idx;
-#endif  // CONFIG_EXT_REFS
   else
     return cpi->alt_fb_idx;
 }
@@ -725,7 +697,6 @@
                                 : NULL;
 }
 
-#if CONFIG_EXT_REFS || CONFIG_TEMPMV_SIGNALING
 static INLINE int enc_is_ref_frame_buf(AV1_COMP *cpi, RefCntBuffer *frame_buf) {
   MV_REFERENCE_FRAME ref_frame;
   AV1_COMMON *const cm = &cpi->common;
@@ -736,7 +707,6 @@
   }
   return (ref_frame <= ALTREF_FRAME);
 }
-#endif  // CONFIG_EXT_REFS
 
 static INLINE unsigned int get_token_alloc(int mb_rows, int mb_cols) {
   // We assume 3 planes all at full resolution. We assume up to 1 token per
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index 0e718b6..9525ad3 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -459,7 +459,6 @@
   cpi->rc.frames_to_key = INT_MAX;
 }
 
-#if CONFIG_EXT_REFS
 static double raw_motion_error_stdev(int *raw_motion_err_list,
                                      int raw_motion_err_counts) {
   int64_t sum_raw_err = 0;
@@ -482,7 +481,6 @@
   raw_err_stdev = sqrt(raw_err_stdev / raw_motion_err_counts);
   return raw_err_stdev;
 }
-#endif  // CONFIG_EXT_REFS
 
 #define UL_INTRA_THRESH 50
 #define INVALID_ROW -1
@@ -534,13 +532,11 @@
   od_adapt_ctx pvq_context;
 #endif
 
-#if CONFIG_EXT_REFS
   int *raw_motion_err_list;
   int raw_motion_err_counts = 0;
   CHECK_MEM_ERROR(
       cm, raw_motion_err_list,
       aom_calloc(cm->mb_rows * cm->mb_cols, sizeof(*raw_motion_err_list)));
-#endif  // CONFIG_EXT_REFS
   // First pass code requires valid last and new frame buffers.
   assert(new_yv12 != NULL);
   assert(frame_is_intra_only(cm) || (lst_yv12 != NULL));
@@ -1006,9 +1002,7 @@
             }
           }
         }
-#if CONFIG_EXT_REFS
         raw_motion_err_list[raw_motion_err_counts++] = raw_motion_error;
-#endif  // CONFIG_EXT_REFS
       } else {
         sr_coded_error += (int64_t)this_error;
       }
@@ -1031,11 +1025,9 @@
 
     aom_clear_system_state();
   }
-#if CONFIG_EXT_REFS
   const double raw_err_stdev =
       raw_motion_error_stdev(raw_motion_err_list, raw_motion_err_counts);
   aom_free(raw_motion_err_list);
-#endif  // CONFIG_EXT_REFS
 
 #if CONFIG_PVQ
 #if !CONFIG_ANS
@@ -1090,9 +1082,7 @@
     fps.intra_skip_pct = (double)intra_skip_count / num_mbs;
     fps.inactive_zone_rows = (double)image_data_start_row;
     fps.inactive_zone_cols = (double)0;  // TODO(paulwilkins): fix
-#if CONFIG_EXT_REFS
     fps.raw_error_stdev = raw_err_stdev;
-#endif  // CONFIG_EXT_REFS
 
     if (mvcount > 0) {
       fps.MVr = (double)sum_mvr / mvcount;
@@ -1144,13 +1134,8 @@
        ((twopass->this_frame_stats.intra_error /
          DOUBLE_DIVIDE_CHECK(twopass->this_frame_stats.coded_error)) > 2.0))) {
     if (gld_yv12 != NULL) {
-#if CONFIG_EXT_REFS
       ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx],
                  cm->ref_frame_map[cpi->lst_fb_idxes[LAST_FRAME - LAST_FRAME]]);
-#else
-      ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx],
-                 cm->ref_frame_map[cpi->lst_fb_idx]);
-#endif  // CONFIG_EXT_REFS
     }
     twopass->sr_update_lag = 1;
   } else {
@@ -1159,26 +1144,16 @@
 
   aom_extend_frame_borders(new_yv12);
 
-// The frame we just compressed now becomes the last frame.
-#if CONFIG_EXT_REFS
+  // The frame we just compressed now becomes the last frame.
   ref_cnt_fb(pool->frame_bufs,
              &cm->ref_frame_map[cpi->lst_fb_idxes[LAST_FRAME - LAST_FRAME]],
              cm->new_fb_idx);
-#else
-  ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->lst_fb_idx],
-             cm->new_fb_idx);
-#endif  // CONFIG_EXT_REFS
 
   // Special case for the first frame. Copy into the GF buffer as a second
   // reference.
   if (cm->current_video_frame == 0 && cpi->gld_fb_idx != INVALID_IDX) {
-#if CONFIG_EXT_REFS
     ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx],
                cm->ref_frame_map[cpi->lst_fb_idxes[LAST_FRAME - LAST_FRAME]]);
-#else
-    ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx],
-               cm->ref_frame_map[cpi->lst_fb_idx]);
-#endif  // CONFIG_EXT_REFS
   }
 
   // Use this to see what the first pass reconstruction looks like.
@@ -1662,21 +1637,6 @@
                 0);
 }
 
-#if !CONFIG_EXT_REFS
-// Current limit on maximum number of active arfs in a GF/ARF group.
-#define MAX_ACTIVE_ARFS 2
-#define ARF_SLOT1 2
-#define ARF_SLOT2 3
-// This function indirects the choice of buffers for arfs.
-// At the moment the values are fixed but this may change as part of
-// the integration process with other codec features that swap buffers around.
-static void get_arf_buffer_indices(unsigned char *arf_buffer_indices) {
-  arf_buffer_indices[0] = ARF_SLOT1;
-  arf_buffer_indices[1] = ARF_SLOT2;
-}
-#endif  // !CONFIG_EXT_REFS
-
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
 // === GF Group of 16 ===
 #define GF_INTERVAL_16 16
@@ -2247,19 +2207,16 @@
   }
 }
 #endif  // USE_GF16_MULTI_LAYER
-#endif  // CONFIG_EXT_REFS
 
 static void define_gf_group_structure(AV1_COMP *cpi) {
   RATE_CONTROL *const rc = &cpi->rc;
 
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
   if (rc->baseline_gf_interval == 16) {
     define_gf_group_structure_16(cpi);
     return;
   }
 #endif  // USE_GF16_MULTI_LAYER
-#endif  // CONFIG_EXT_REFS
 
   TWO_PASS *const twopass = &cpi->twopass;
   GF_GROUP *const gf_group = &twopass->gf_group;
@@ -2267,7 +2224,6 @@
   int frame_index = 0;
   const int key_frame = cpi->common.frame_type == KEY_FRAME;
 
-#if CONFIG_EXT_REFS
   // The use of bi-predictive frames are only enabled when following 3
   // conditions are met:
   // (1) ALTREF is enabled;
@@ -2288,14 +2244,6 @@
   int subgroup_interval[MAX_EXT_ARFS + 1];
   int is_sg_bipred_enabled = is_bipred_enabled;
   int accumulative_subgroup_interval = 0;
-#else
-  int mid_frame_idx;
-  unsigned char arf_buffer_indices[MAX_ACTIVE_ARFS];
-#endif  // CONFIG_EXT_REFS
-
-#if !CONFIG_EXT_REFS
-  get_arf_buffer_indices(arf_buffer_indices);
-#endif  // !CONFIG_EXT_REFS
 
   // For key frames the frame target rate is already set and it
   // is also the golden frame.
@@ -2308,25 +2256,16 @@
       gf_group->update_type[frame_index] = GF_UPDATE;
       gf_group->rf_level[frame_index] = GF_ARF_STD;
     }
-#if CONFIG_EXT_REFS
     gf_group->arf_update_idx[frame_index] = 0;
     gf_group->arf_ref_idx[frame_index] = 0;
-#else
-    gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0];
-    gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0];
-#endif  // CONFIG_EXT_REFS
   }
 
-#if CONFIG_EXT_REFS
   gf_group->bidir_pred_enabled[frame_index] = 0;
   gf_group->brf_src_offset[frame_index] = 0;
-#endif  // CONFIG_EXT_REFS
 
   frame_index++;
 
-#if CONFIG_EXT_REFS
   bipred_frame_index++;
-#endif  // CONFIG_EXT_REFS
 
   // === [frame_index == 1] ===
   if (rc->source_alt_ref_pending) {
@@ -2335,21 +2274,13 @@
     gf_group->arf_src_offset[frame_index] =
         (unsigned char)(rc->baseline_gf_interval - 1);
 
-#if CONFIG_EXT_REFS
     gf_group->arf_update_idx[frame_index] = 0;
     gf_group->arf_ref_idx[frame_index] = 0;
 
     gf_group->bidir_pred_enabled[frame_index] = 0;
     gf_group->brf_src_offset[frame_index] = 0;
-// NOTE: "bidir_pred_frame_index" stays unchanged for ARF_UPDATE frames.
-#else
-    gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0];
-    gf_group->arf_ref_idx[frame_index] =
-        arf_buffer_indices[cpi->multi_arf_last_grp_enabled &&
-                           rc->source_alt_ref_active];
-#endif  // CONFIG_EXT_REFS
+    // NOTE: "bidir_pred_frame_index" stays unchanged for ARF_UPDATE frames.
 
-#if CONFIG_EXT_REFS
     // Work out the ARFs' positions in this gf group
     // NOTE(weitinglin): ALT_REFs' are indexed inversely, but coded in display
     // order (except for the original ARF). In the example of three ALT_REF's,
@@ -2370,11 +2301,9 @@
     subgroup_interval[cpi->num_extra_arfs] =
         cpi->arf_pos_for_ovrly[cpi->num_extra_arfs] - frame_index -
         (cpi->num_extra_arfs == 0 ? 1 : 2);
-#endif  // CONFIG_EXT_REFS
 
     ++frame_index;
 
-#if CONFIG_EXT_REFS
     // Insert an extra ARF
     // === [frame_index == 2] ===
     if (cpi->num_extra_arfs) {
@@ -2387,43 +2316,12 @@
       ++frame_index;
     }
     accumulative_subgroup_interval += subgroup_interval[cpi->num_extra_arfs];
-#else   // !CONFIG_EXT_ARFS
-    if (cpi->multi_arf_enabled) {
-      // Set aside a slot for a level 1 arf.
-      gf_group->update_type[frame_index] = ARF_UPDATE;
-      gf_group->rf_level[frame_index] = GF_ARF_LOW;
-      gf_group->arf_src_offset[frame_index] =
-          (unsigned char)((rc->baseline_gf_interval >> 1) - 1);
-      gf_group->arf_update_idx[frame_index] = arf_buffer_indices[1];
-      gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0];
-      ++frame_index;
-    }
-#endif  // CONFIG_EXT_ARFS
   }
 
-#if !CONFIG_EXT_REFS
-  // Define middle frame
-  mid_frame_idx = frame_index + (rc->baseline_gf_interval >> 1) - 1;
-#endif  // !CONFIG_EXT_REFS
-
   for (i = 0; i < rc->baseline_gf_interval - rc->source_alt_ref_pending; ++i) {
-#if !CONFIG_EXT_REFS
-    int arf_idx = 0;
-
-    if (rc->source_alt_ref_pending && cpi->multi_arf_enabled) {
-      if (frame_index <= mid_frame_idx) arf_idx = 1;
-    }
-#endif  // !CONFIG_EXT_REFS
-
-#if CONFIG_EXT_REFS
     gf_group->arf_update_idx[frame_index] = which_arf;
     gf_group->arf_ref_idx[frame_index] = which_arf;
-#else
-    gf_group->arf_update_idx[frame_index] = arf_buffer_indices[arf_idx];
-    gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[arf_idx];
-#endif  // CONFIG_EXT_REFS
 
-#if CONFIG_EXT_REFS
     // If we are going to have ARFs, check whether we can have BWDREF in this
     // subgroup, and further, whether we can have ARF subgroup which contains
     // the BWDREF subgroup but contained within the GF group:
@@ -2472,18 +2370,14 @@
         bipred_group_end = 1;
       }
     } else {
-#endif  // CONFIG_EXT_REFS
       gf_group->update_type[frame_index] = LF_UPDATE;
       gf_group->rf_level[frame_index] = INTER_NORMAL;
-#if CONFIG_EXT_REFS
       gf_group->bidir_pred_enabled[frame_index] = 0;
       gf_group->brf_src_offset[frame_index] = 0;
     }
-#endif  // CONFIG_EXT_REFS
 
     ++frame_index;
 
-#if CONFIG_EXT_REFS
     // Check if we need to update the ARF.
     if (is_sg_bipred_enabled && cpi->num_extra_arfs && which_arf > 0 &&
         frame_index > cpi->arf_pos_for_ovrly[which_arf]) {
@@ -2503,25 +2397,19 @@
         ++frame_index;
       }
     }
-#endif  // CONFIG_EXT_REFS
   }
 
-// NOTE: We need to configure the frame at the end of the sequence + 1 that will
-//       be the start frame for the next group. Otherwise prior to the call to
-//       av1_rc_get_second_pass_params() the data will be undefined.
-#if CONFIG_EXT_REFS
+  // NOTE: We need to configure the frame at the end of the sequence + 1 that
+  // will
+  //       be the start frame for the next group. Otherwise prior to the call to
+  //       av1_rc_get_second_pass_params() the data will be undefined.
   gf_group->arf_update_idx[frame_index] = 0;
   gf_group->arf_ref_idx[frame_index] = 0;
-#else
-  gf_group->arf_update_idx[frame_index] = arf_buffer_indices[0];
-  gf_group->arf_ref_idx[frame_index] = arf_buffer_indices[0];
-#endif  // CONFIG_EXT_REFS
 
   if (rc->source_alt_ref_pending) {
     gf_group->update_type[frame_index] = OVERLAY_UPDATE;
     gf_group->rf_level[frame_index] = INTER_NORMAL;
 
-#if CONFIG_EXT_REFS
     cpi->arf_pos_in_gf[0] = 1;
     if (cpi->num_extra_arfs) {
       // Overwrite the update_type for extra-ARF's corresponding internal
@@ -2534,21 +2422,13 @@
         gf_group->rf_level[cpi->arf_pos_for_ovrly[i]] = INTER_NORMAL;
       }
     }
-#else
-    // Final setup for second arf and its overlay.
-    if (cpi->multi_arf_enabled) {
-      gf_group->update_type[mid_frame_idx] = OVERLAY_UPDATE;
-    }
-#endif  // CONFIG_EXT_REFS
   } else {
     gf_group->update_type[frame_index] = GF_UPDATE;
     gf_group->rf_level[frame_index] = GF_ARF_STD;
   }
 
-#if CONFIG_EXT_REFS
   gf_group->bidir_pred_enabled[frame_index] = 0;
   gf_group->brf_src_offset[frame_index] = 0;
-#endif  // CONFIG_EXT_REFS
 }
 
 static void allocate_gf_group_bits(AV1_COMP *cpi, int64_t gf_group_bits,
@@ -2567,17 +2447,11 @@
   double modified_err = 0.0;
   double err_fraction;
   int mid_boost_bits = 0;
-#if CONFIG_EXT_REFS
   int ext_arf_boost[MAX_EXT_ARFS];
-#else
-  int mid_frame_idx;
-#endif  // CONFIG_EXT_REFS
 
   define_gf_group_structure(cpi);
 
-#if CONFIG_EXT_REFS
   av1_zero_array(ext_arf_boost, MAX_EXT_ARFS);
-#endif  // CONFIG_EXT_REFS
 
   key_frame = cpi->common.frame_type == KEY_FRAME;
 
@@ -2607,24 +2481,14 @@
 
     ++frame_index;
 
-#if CONFIG_EXT_REFS
     // Skip all the extra-ARF's right after ARF at the starting segment of
     // the current GF group.
     if (cpi->num_extra_arfs) {
       while (gf_group->update_type[frame_index] == INTNL_ARF_UPDATE)
         ++frame_index;
     }
-#else   // !CONFIG_EXT_ARFS
-    // Set aside a slot for a level 1 arf.
-    if (cpi->multi_arf_enabled) ++frame_index;
-#endif  // CONFIG_EXT_ARFS
   }
 
-#if !CONFIG_EXT_REFS
-  // Define middle frame
-  mid_frame_idx = frame_index + (rc->baseline_gf_interval >> 1) - 1;
-#endif  // !CONFIG_EXT_REFS
-
   // Allocate bits to the other frames in the group.
   for (i = 0; i < rc->baseline_gf_interval - rc->source_alt_ref_pending; ++i) {
     if (EOF == input_stats(twopass, &frame_stats)) break;
@@ -2646,7 +2510,6 @@
     target_frame_size =
         clamp(target_frame_size, 0, AOMMIN(max_bits, (int)total_group_bits));
 
-#if CONFIG_EXT_REFS
     if (gf_group->update_type[frame_index] == BRF_UPDATE) {
       // Boost up the allocated bits on BWDREF_FRAME
       gf_group->bit_allocation[frame_index] =
@@ -2662,28 +2525,22 @@
     } else {
       assert(gf_group->update_type[frame_index] == LF_UPDATE ||
              gf_group->update_type[frame_index] == INTNL_OVERLAY_UPDATE);
-#endif  // CONFIG_EXT_REFS
       gf_group->bit_allocation[frame_index] = target_frame_size;
-#if CONFIG_EXT_REFS
     }
-#endif  // CONFIG_EXT_REFS
 
     ++frame_index;
 
-#if CONFIG_EXT_REFS
     // Skip all the extra-ARF's.
     if (cpi->num_extra_arfs) {
       while (gf_group->update_type[frame_index] == INTNL_ARF_UPDATE)
         ++frame_index;
     }
-#endif  // CONFIG_EXT_REFS
   }
 
   // NOTE: We need to configure the frame at the end of the sequence + 1 that
   //       will be the start frame for the next group. Otherwise prior to the
   //       call to av1_rc_get_second_pass_params() the data will be undefined.
   if (rc->source_alt_ref_pending) {
-#if CONFIG_EXT_REFS
     if (cpi->num_extra_arfs) {
       // NOTE: For bit allocation, move the allocated bits associated with
       //       INTNL_OVERLAY_UPDATE to the corresponding INTNL_ARF_UPDATE.
@@ -2702,14 +2559,6 @@
         gf_group->bit_allocation[cpi->arf_pos_for_ovrly[i]] = 0;
       }
     }
-#else
-    // Final setup for second arf and its overlay.
-    if (cpi->multi_arf_enabled) {
-      gf_group->bit_allocation[2] =
-          gf_group->bit_allocation[mid_frame_idx] + mid_boost_bits;
-      gf_group->bit_allocation[mid_frame_idx] = 0;
-    }
-#endif  // CONFIG_EXT_REFS
   }
 
   // Note whether multi-arf was enabled this group for next time.
@@ -2761,10 +2610,8 @@
   const int is_key_frame = frame_is_intra_only(cm);
   const int arf_active_or_kf = is_key_frame || rc->source_alt_ref_active;
 
-#if CONFIG_EXT_REFS
   cpi->extra_arf_allowed = 1;
   cpi->bwd_ref_allowed = 1;
-#endif  // CONFIG_EXT_REFS
 
   // Reset the GF group data structures unless this is a key
   // frame in which case it will already have been done.
@@ -2826,11 +2673,9 @@
     }
   }
 
-#if CONFIG_EXT_REFS || CONFIG_BGSPRITE
   double avg_sr_coded_error = 0;
   double avg_raw_err_stdev = 0;
   int non_zero_stdev_count = 0;
-#endif  // CONFIG_EXT_REFS || CONFIG_BGSPRITE
 #if CONFIG_BGSPRITE
   double avg_pcnt_second_ref = 0;
   int non_zero_pcnt_second_ref_count = 0;
@@ -2859,14 +2704,12 @@
     accumulate_frame_motion_stats(
         &next_frame, &this_frame_mv_in_out, &mv_in_out_accumulator,
         &abs_mv_in_out_accumulator, &mv_ratio_accumulator);
-#if CONFIG_EXT_REFS || CONFIG_BGSPRITE
     // sum up the metric values of current gf group
     avg_sr_coded_error += next_frame.sr_coded_error;
     if (fabs(next_frame.raw_error_stdev) > 0.000001) {
       non_zero_stdev_count++;
       avg_raw_err_stdev += next_frame.raw_error_stdev;
     }
-#endif  // CONFIG_EXT_REFS || CONFIG_BGSPRITE
 #if CONFIG_BGSPRITE
     if (this_frame->pcnt_second_ref) {
       avg_pcnt_second_ref += this_frame->pcnt_second_ref;
@@ -2912,18 +2755,14 @@
              (abs_mv_in_out_accumulator > 3.0) ||
              (mv_in_out_accumulator < -2.0) ||
              ((boost_score - old_boost_score) < BOOST_BREAKOUT)))) {
-#if CONFIG_EXT_REFS
       // If GF group interval is < 12, we force it to be 8. Otherwise,
       // if it is >= 12, we keep it as is.
       // NOTE: 'i' is 1 more than the GF group interval candidate that is being
       //       checked.
       if (i == (8 + 1) || i >= (12 + 1)) {
-#endif  // CONFIG_EXT_REFS
         boost_score = old_boost_score;
         break;
-#if CONFIG_EXT_REFS
       }
-#endif  // CONFIG_EXT_REFS
     }
 
     *this_frame = next_frame;
@@ -2934,12 +2773,10 @@
   // Was the group length constrained by the requirement for a new KF?
   rc->constrained_gf_group = (i >= rc->frames_to_key) ? 1 : 0;
 
-#if CONFIG_EXT_REFS || CONFIG_BGSPRITE
   const int num_mbs = (cpi->oxcf.resize_mode != RESIZE_NONE) ? cpi->initial_mbs
                                                              : cpi->common.MBs;
   assert(num_mbs > 0);
   if (i) avg_sr_coded_error /= i;
-#endif  // CONFIG_EXT_REFS || CONFIG_BGSPRITE
 
   // Should we use the alternate reference frame.
   if (allow_alt_ref && (i < cpi->oxcf.lag_in_frames) &&
@@ -2973,7 +2810,6 @@
 
   // Set the interval until the next gf.
   rc->baseline_gf_interval = i - (is_key_frame || rc->source_alt_ref_pending);
-#if CONFIG_EXT_REFS
   if (non_zero_stdev_count) avg_raw_err_stdev /= non_zero_stdev_count;
 
   // Disable extra altrefs and backward refs for "still" gf group:
@@ -2998,15 +2834,12 @@
   }
   // Currently at maximum two extra ARFs' are allowed
   assert(cpi->num_extra_arfs <= MAX_EXT_ARFS);
-#endif  // CONFIG_EXT_REFS
 
   rc->frames_till_gf_update_due = rc->baseline_gf_interval;
 
-#if CONFIG_EXT_REFS
   rc->bipred_group_interval = BFG_INTERVAL;
   // The minimum bi-predictive frame group interval is 2.
   if (rc->bipred_group_interval < 2) rc->bipred_group_interval = 0;
-#endif  // CONFIG_EXT_REFS
 
   // Reset the file position.
   reset_fpf_position(twopass, start_pos);
@@ -3424,7 +3257,7 @@
   twopass->modified_error_left -= kf_group_err;
 }
 
-#if CONFIG_EXT_REFS && USE_GF16_MULTI_LAYER
+#if USE_GF16_MULTI_LAYER
 // === GF Group of 16 ===
 void av1_ref_frame_map_idx_updates(AV1_COMP *cpi, int gf_frame_index) {
   TWO_PASS *const twopass = &cpi->twopass;
@@ -3569,7 +3402,7 @@
     default: break;
   }
 }
-#endif  // CONFIG_EXT_REFS && USE_GF16_MULTI_LAYER
+#endif  // USE_GF16_MULTI_LAYER
 
 // Define the reference buffers that will be updated post encode.
 static void configure_buffer_updates(AV1_COMP *cpi) {
@@ -3579,7 +3412,6 @@
   // cpi->rc.is_$Source_Type to make this function as it is in the comment?
 
   cpi->rc.is_src_frame_alt_ref = 0;
-#if CONFIG_EXT_REFS
   cpi->rc.is_bwd_ref_frame = 0;
   cpi->rc.is_last_bipred_frame = 0;
   cpi->rc.is_bipred_frame = 0;
@@ -3592,22 +3424,21 @@
     return;
   }
 #endif  // USE_GF16_MULTI_LAYER
-#endif  // CONFIG_EXT_REFS
 
   switch (twopass->gf_group.update_type[twopass->gf_group.index]) {
-    case KF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 1;
-#if CONFIG_EXT_REFS
+    case KF_UPDATE:
+      cpi->refresh_last_frame = 1;
+      cpi->refresh_golden_frame = 1;
       cpi->refresh_bwd_ref_frame = 1;
       cpi->refresh_alt2_ref_frame = 1;
-#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 1;
       break;
 
-    case LF_UPDATE: cpi->refresh_last_frame = 1; cpi->refresh_golden_frame = 0;
-#if CONFIG_EXT_REFS
+    case LF_UPDATE:
+      cpi->refresh_last_frame = 1;
+      cpi->refresh_golden_frame = 0;
       cpi->refresh_bwd_ref_frame = 0;
       cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
 
@@ -3616,35 +3447,30 @@
       //               needed.
       cpi->refresh_last_frame = 1;
       cpi->refresh_golden_frame = 1;
-#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
       cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
 
     case OVERLAY_UPDATE:
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 1;
-#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
       cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
 
       cpi->rc.is_src_frame_alt_ref = 1;
       break;
 
-    case ARF_UPDATE: cpi->refresh_last_frame = 0; cpi->refresh_golden_frame = 0;
-#if CONFIG_EXT_REFS
+    case ARF_UPDATE:
+      cpi->refresh_last_frame = 0;
+      cpi->refresh_golden_frame = 0;
       // NOTE: BWDREF does not get updated along with ALTREF_FRAME.
       cpi->refresh_bwd_ref_frame = 0;
       cpi->refresh_alt2_ref_frame = 0;
-#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 1;
       break;
 
-#if CONFIG_EXT_REFS
     case BRF_UPDATE:
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 0;
@@ -3693,7 +3519,6 @@
       cpi->refresh_alt2_ref_frame = 1;
       cpi->refresh_alt_ref_frame = 0;
       break;
-#endif  // CONFIG_EXT_REFS
 
     default: assert(0); break;
   }
@@ -3734,11 +3559,8 @@
 
   // If this is an arf frame then we dont want to read the stats file or
   // advance the input pointer as we already have what we need.
-  if (gf_group->update_type[gf_group->index] == ARF_UPDATE
-#if CONFIG_EXT_REFS
-      || gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE
-#endif  // CONFIG_EXT_REFS
-      ) {
+  if (gf_group->update_type[gf_group->index] == ARF_UPDATE ||
+      gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) {
     configure_buffer_updates(cpi);
     target_rate = gf_group->bit_allocation[gf_group->index];
     target_rate = av1_rc_clamp_pframe_target_size(cpi, target_rate);
diff --git a/av1/encoder/firstpass.h b/av1/encoder/firstpass.h
index 9ac542b..ee21b65 100644
--- a/av1/encoder/firstpass.h
+++ b/av1/encoder/firstpass.h
@@ -42,7 +42,6 @@
 } FIRSTPASS_MB_STATS;
 #endif
 
-#if CONFIG_EXT_REFS
 // Length of the bi-predictive frame group (BFG)
 // NOTE: Currently each BFG contains one backward ref (BWF) frame plus a certain
 //       number of bi-predictive frames.
@@ -64,7 +63,6 @@
 #define MAX_SR_CODED_ERROR 40
 #define MAX_RAW_ERR_VAR 2000
 #define MIN_MV_IN_OUT 0.4
-#endif  // CONFIG_EXT_REFS
 
 #define VLOW_MOTION_THRESHOLD 950
 
@@ -91,10 +89,8 @@
   double new_mv_count;
   double duration;
   double count;
-#if CONFIG_EXT_REFS || CONFIG_BGSPRITE
   // standard deviation for (0, 0) motion prediction error
   double raw_error_stdev;
-#endif  // CONFIG_EXT_REFS
 } FIRSTPASS_STATS;
 
 typedef enum {
@@ -103,16 +99,12 @@
   GF_UPDATE = 2,
   ARF_UPDATE = 3,
   OVERLAY_UPDATE = 4,
-#if CONFIG_EXT_REFS
   BRF_UPDATE = 5,            // Backward Reference Frame
   LAST_BIPRED_UPDATE = 6,    // Last Bi-predictive Frame
   BIPRED_UPDATE = 7,         // Bi-predictive Frame, but not the last one
   INTNL_OVERLAY_UPDATE = 8,  // Internal Overlay Frame
   INTNL_ARF_UPDATE = 9,      // Internal Altref Frame (candidate for ALTREF2)
   FRAME_UPDATE_TYPES = 10
-#else   // !CONFIG_EXT_REFS
-  FRAME_UPDATE_TYPES = 5
-#endif  // CONFIG_EXT_REFS
 } FRAME_UPDATE_TYPE;
 
 #define FC_ANIMATION_THRESH 0.15
@@ -129,13 +121,11 @@
   unsigned char arf_src_offset[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char arf_update_idx[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char arf_ref_idx[(MAX_LAG_BUFFERS * 2) + 1];
-#if CONFIG_EXT_REFS
   unsigned char brf_src_offset[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char bidir_pred_enabled[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char ref_fb_idx_map[(MAX_LAG_BUFFERS * 2) + 1][REF_FRAMES];
   unsigned char refresh_idx[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char refresh_flag[(MAX_LAG_BUFFERS * 2) + 1];
-#endif  // CONFIG_EXT_REFS
   int bit_allocation[(MAX_LAG_BUFFERS * 2) + 1];
 } GF_GROUP;
 
@@ -198,7 +188,6 @@
 // Post encode update of the rate control parameters for 2-pass
 void av1_twopass_postencode_update(struct AV1_COMP *cpi);
 
-#if CONFIG_EXT_REFS
 #if USE_GF16_MULTI_LAYER
 void av1_ref_frame_map_idx_updates(struct AV1_COMP *cpi, int gf_frame_index);
 #endif  // USE_GF16_MULTI_LAYER
@@ -213,7 +202,6 @@
   else
     return 0;
 }
-#endif  // CONFIG_EXT_REFS
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index a90cb88..c431f65 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -196,12 +196,8 @@
   const AV1EncoderConfig *oxcf = &cpi->oxcf;
   const int min_frame_target =
       AOMMAX(rc->min_frame_bandwidth, rc->avg_frame_bandwidth >> 5);
-// Clip the frame target to the minimum setup value.
-#if CONFIG_EXT_REFS
+  // Clip the frame target to the minimum setup value.
   if (cpi->rc.is_src_frame_alt_ref) {
-#else
-  if (cpi->refresh_golden_frame && rc->is_src_frame_alt_ref) {
-#endif  // CONFIG_EXT_REFS
     // If there is an active ARF at this location use the minimum
     // bits on this frame even if it is a constructed arf.
     // The active maximum quantizer insures that an appropriate
@@ -239,14 +235,10 @@
   const AV1_COMMON *const cm = &cpi->common;
   RATE_CONTROL *const rc = &cpi->rc;
 
-// Non-viewable frames are a special case and are treated as pure overhead.
-#if CONFIG_EXT_REFS
+  // Non-viewable frames are a special case and are treated as pure overhead.
   // TODO(zoeliu): To further explore whether we should treat BWDREF_FRAME
   //               differently, since it is a no-show frame.
   if (!cm->show_frame && !rc->is_bwd_ref_frame)
-#else
-  if (!cm->show_frame)
-#endif  // CONFIG_EXT_REFS
     rc->bits_off_target -= encoded_frame_size;
   else
     rc->bits_off_target += rc->avg_frame_bandwidth - encoded_frame_size;
@@ -590,11 +582,9 @@
     active_worst_quality =
         curr_frame == 0 ? rc->worst_quality : rc->last_q[KEY_FRAME] * 2;
   } else {
-    if (!rc->is_src_frame_alt_ref && (cpi->refresh_golden_frame ||
-#if CONFIG_EXT_REFS
-                                      cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
-                                      cpi->refresh_alt_ref_frame)) {
+    if (!rc->is_src_frame_alt_ref &&
+        (cpi->refresh_golden_frame || cpi->refresh_alt2_ref_frame ||
+         cpi->refresh_alt_ref_frame)) {
       active_worst_quality = curr_frame == 1 ? rc->last_q[KEY_FRAME] * 5 / 4
                                              : rc->last_q[INTER_FRAME];
     } else {
@@ -933,24 +923,15 @@
 int av1_frame_type_qdelta(const AV1_COMP *cpi, int rf_level, int q) {
   static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = {
     1.00,  // INTER_NORMAL
-#if CONFIG_EXT_REFS
     0.80,  // INTER_LOW
     1.50,  // INTER_HIGH
     1.25,  // GF_ARF_LOW
-#else
-    1.00,  // INTER_HIGH
-    1.50,  // GF_ARF_LOW
-#endif     // CONFIG_EXT_REFS
     2.00,  // GF_ARF_STD
     2.00,  // KF_STD
   };
-  static const FRAME_TYPE frame_type[RATE_FACTOR_LEVELS] =
-#if CONFIG_EXT_REFS
-      { INTER_FRAME, INTER_FRAME, INTER_FRAME,
-        INTER_FRAME, INTER_FRAME, KEY_FRAME };
-#else
-      { INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, KEY_FRAME };
-#endif  // CONFIG_EXT_REFS
+  static const FRAME_TYPE frame_type[RATE_FACTOR_LEVELS] = {
+    INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, KEY_FRAME
+  };
   const AV1_COMMON *const cm = &cpi->common;
   int qdelta =
       av1_compute_qdelta_by_rate(&cpi->rc, frame_type[rf_level], q,
@@ -1020,11 +1001,9 @@
       active_best_quality +=
           av1_compute_qdelta(rc, q_val, q_val * q_adj_factor, cm->bit_depth);
     }
-  } else if (!rc->is_src_frame_alt_ref && (cpi->refresh_golden_frame ||
-#if CONFIG_EXT_REFS
-                                           cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
-                                           cpi->refresh_alt_ref_frame)) {
+  } else if (!rc->is_src_frame_alt_ref &&
+             (cpi->refresh_golden_frame || cpi->refresh_alt2_ref_frame ||
+              cpi->refresh_alt_ref_frame)) {
     // Use the lower of active_worst_quality and recent
     // average Q as basis for GF/ARF best Q limit unless last frame was
     // a key frame.
@@ -1044,11 +1023,7 @@
       active_best_quality = active_best_quality * 15 / 16;
 
     } else if (oxcf->rc_mode == AOM_Q) {
-#if CONFIG_EXT_REFS
       if (!cpi->refresh_alt_ref_frame && !cpi->refresh_alt2_ref_frame) {
-#else
-      if (!cpi->refresh_alt_ref_frame) {
-#endif  // CONFIG_EXT_REFS
         active_best_quality = cq_level;
       } else {
         active_best_quality = get_gf_active_quality(rc, q, cm->bit_depth);
@@ -1080,11 +1055,9 @@
   if ((cpi->oxcf.rc_mode != AOM_Q) &&
       (cpi->twopass.gf_zeromotion_pct < VLOW_MOTION_THRESHOLD)) {
     if (frame_is_intra_only(cm) ||
-        (!rc->is_src_frame_alt_ref && (cpi->refresh_golden_frame ||
-#if CONFIG_EXT_REFS
-                                       cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
-                                       cpi->refresh_alt_ref_frame))) {
+        (!rc->is_src_frame_alt_ref &&
+         (cpi->refresh_golden_frame || cpi->refresh_alt2_ref_frame ||
+          cpi->refresh_alt_ref_frame))) {
       active_best_quality -=
           (cpi->twopass.extend_minq + cpi->twopass.extend_minq_fast);
       active_worst_quality += (cpi->twopass.extend_maxq / 2);
@@ -1217,21 +1190,13 @@
 static void update_golden_frame_stats(AV1_COMP *cpi) {
   RATE_CONTROL *const rc = &cpi->rc;
 
-#if CONFIG_EXT_REFS
   // Update the Golden frame usage counts.
   // NOTE(weitinglin): If we use show_existing_frame for an OVERLAY frame,
   //                   only the virtual indices for the reference frame will be
   //                   updated and cpi->refresh_golden_frame will still be zero.
   if (cpi->refresh_golden_frame || rc->is_src_frame_alt_ref) {
-#else   // !CONFIG_EXT_REFS
-  // Update the Golden frame usage counts.
-  if (cpi->refresh_golden_frame) {
-#endif  // CONFIG_EXT_REFS
-
-#if CONFIG_EXT_REFS
     // We will not use internal overlay frames to replace the golden frame
     if (!rc->is_src_frame_ext_arf)
-#endif  // CONFIG_EXT_REFS
       // this frame refreshes means next frames don't unless specified by user
       rc->frames_since_golden = 0;
 
@@ -1248,11 +1213,7 @@
     // Decrement count down till next gf
     if (rc->frames_till_gf_update_due > 0) rc->frames_till_gf_update_due--;
 
-#if CONFIG_EXT_REFS
   } else if (!cpi->refresh_alt_ref_frame && !cpi->refresh_alt2_ref_frame) {
-#else
-  } else if (!cpi->refresh_alt_ref_frame) {
-#endif  // CONFIG_EXT_REFS
     // Decrement count down till next gf
     if (rc->frames_till_gf_update_due > 0) rc->frames_till_gf_update_due--;
 
@@ -1282,10 +1243,7 @@
         ROUND_POWER_OF_TWO(3 * rc->avg_frame_qindex[KEY_FRAME] + qindex, 2);
   } else {
     if (!rc->is_src_frame_alt_ref &&
-        !(cpi->refresh_golden_frame ||
-#if CONFIG_EXT_REFS
-          cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
+        !(cpi->refresh_golden_frame || cpi->refresh_alt2_ref_frame ||
           cpi->refresh_alt_ref_frame)) {
       rc->last_q[INTER_FRAME] = qindex;
       rc->avg_frame_qindex[INTER_FRAME] =
@@ -1307,10 +1265,7 @@
   // This is used to help set quality in forced key frames to reduce popping
   if ((qindex < rc->last_boosted_qindex) || (cm->frame_type == KEY_FRAME) ||
       (!rc->constrained_gf_group &&
-       (cpi->refresh_alt_ref_frame ||
-#if CONFIG_EXT_REFS
-        cpi->refresh_alt2_ref_frame ||
-#endif  // CONFIG_EXT_REFS
+       (cpi->refresh_alt_ref_frame || cpi->refresh_alt2_ref_frame ||
         (cpi->refresh_golden_frame && !rc->is_src_frame_alt_ref)))) {
     rc->last_boosted_qindex = qindex;
   }
@@ -1337,14 +1292,10 @@
 
   // Actual bits spent
   rc->total_actual_bits += rc->projected_frame_size;
-#if CONFIG_EXT_REFS
   // TODO(zoeliu): To investigate whether we should treat BWDREF_FRAME
   //               differently here for rc->avg_frame_bandwidth.
   rc->total_target_bits +=
       (cm->show_frame || rc->is_bwd_ref_frame) ? rc->avg_frame_bandwidth : 0;
-#else
-  rc->total_target_bits += cm->show_frame ? rc->avg_frame_bandwidth : 0;
-#endif  // CONFIG_EXT_REFS
 
   rc->total_target_vs_actual = rc->total_actual_bits - rc->total_target_bits;
 
@@ -1358,13 +1309,9 @@
 
   if (cm->frame_type == KEY_FRAME) rc->frames_since_key = 0;
 
-#if CONFIG_EXT_REFS
   // TODO(zoeliu): To investigate whether we should treat BWDREF_FRAME
   //               differently here for rc->avg_frame_bandwidth.
   if (cm->show_frame || rc->is_bwd_ref_frame) {
-#else
-  if (cm->show_frame) {
-#endif  // CONFIG_EXT_REFS
     rc->frames_since_key++;
     rc->frames_to_key--;
   }
diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h
index 8b410e7..6562de1 100644
--- a/av1/encoder/ratectrl.h
+++ b/av1/encoder/ratectrl.h
@@ -28,7 +28,6 @@
 #define MAX_GF_INTERVAL 16
 #define FIXED_GF_INTERVAL 8  // Used in some testing modes only
 
-#if CONFIG_EXT_REFS
 typedef enum {
   INTER_NORMAL = 0,
   INTER_LOW = 1,
@@ -38,16 +37,6 @@
   KF_STD = 5,
   RATE_FACTOR_LEVELS = 6
 } RATE_FACTOR_LEVEL;
-#else
-typedef enum {
-  INTER_NORMAL = 0,
-  INTER_HIGH = 1,
-  GF_ARF_LOW = 2,
-  GF_ARF_STD = 3,
-  KF_STD = 4,
-  RATE_FACTOR_LEVELS = 5
-} RATE_FACTOR_LEVEL;
-#endif  // CONFIG_EXT_REFS
 
 typedef struct {
   int resize_width;
@@ -89,7 +78,6 @@
   int source_alt_ref_active;
   int is_src_frame_alt_ref;
 
-#if CONFIG_EXT_REFS
   // Length of the bi-predictive frame group interval
   int bipred_group_interval;
 
@@ -99,7 +87,6 @@
   int is_last_bipred_frame;
   int is_bipred_frame;
   int is_src_frame_ext_arf;
-#endif  // CONFIG_EXT_REFS
 
   int avg_frame_bandwidth;  // Average frame size target for clip
   int min_frame_bandwidth;  // Minimum allocation used for any frame
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index cac0105..e9e68ad 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -404,14 +404,12 @@
                                          8,  8,  4,  4,  2,  2,  1,  0 };
 static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
   128, 144, 128, 128, 144,
-#if CONFIG_EXT_REFS
   // TODO(zoeliu): To adjust further following factor values.
   128, 128, 128,
   // TODO(weitinglin): We should investigate if the values should be the same
   //                   as the value used by OVERLAY frame
   144,  // INTNL_OVERLAY_UPDATE
   128   // INTNL_ARF_UPDATE
-#endif  // CONFIG_EXT_REFS
 };
 
 int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) {
@@ -1273,22 +1271,18 @@
 
   if (sf->adaptive_rd_thresh) {
     rd->thresh_mult[THR_NEARESTMV] = 300;
-#if CONFIG_EXT_REFS
     rd->thresh_mult[THR_NEARESTL2] = 300;
     rd->thresh_mult[THR_NEARESTL3] = 300;
     rd->thresh_mult[THR_NEARESTB] = 300;
     rd->thresh_mult[THR_NEARESTA2] = 300;
-#endif  // CONFIG_EXT_REFS
     rd->thresh_mult[THR_NEARESTA] = 300;
     rd->thresh_mult[THR_NEARESTG] = 300;
   } else {
     rd->thresh_mult[THR_NEARESTMV] = 0;
-#if CONFIG_EXT_REFS
     rd->thresh_mult[THR_NEARESTL2] = 0;
     rd->thresh_mult[THR_NEARESTL3] = 0;
     rd->thresh_mult[THR_NEARESTB] = 0;
     rd->thresh_mult[THR_NEARESTA2] = 0;
-#endif  // CONFIG_EXT_REFS
     rd->thresh_mult[THR_NEARESTA] = 0;
     rd->thresh_mult[THR_NEARESTG] = 0;
   }
@@ -1296,32 +1290,26 @@
   rd->thresh_mult[THR_DC] += 1000;
 
   rd->thresh_mult[THR_NEWMV] += 1000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEWL2] += 1000;
   rd->thresh_mult[THR_NEWL3] += 1000;
   rd->thresh_mult[THR_NEWB] += 1000;
   rd->thresh_mult[THR_NEWA2] = 1000;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEWA] += 1000;
   rd->thresh_mult[THR_NEWG] += 1000;
 
   rd->thresh_mult[THR_NEARMV] += 1000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEARL2] += 1000;
   rd->thresh_mult[THR_NEARL3] += 1000;
   rd->thresh_mult[THR_NEARB] += 1000;
   rd->thresh_mult[THR_NEARA2] = 1000;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEARA] += 1000;
   rd->thresh_mult[THR_NEARG] += 1000;
 
   rd->thresh_mult[THR_ZEROMV] += 2000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_ZEROL2] += 2000;
   rd->thresh_mult[THR_ZEROL3] += 2000;
   rd->thresh_mult[THR_ZEROB] += 2000;
   rd->thresh_mult[THR_ZEROA2] = 2000;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_ZEROG] += 2000;
   rd->thresh_mult[THR_ZEROA] += 2000;
 
@@ -1329,59 +1317,46 @@
 
 #if CONFIG_COMPOUND_SINGLEREF
   rd->thresh_mult[THR_SR_NEAREST_NEARMV] += 1200;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEAREST_NEARL2] += 1200;
   rd->thresh_mult[THR_SR_NEAREST_NEARL3] += 1200;
   rd->thresh_mult[THR_SR_NEAREST_NEARB] += 1200;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEAREST_NEARA] += 1200;
   rd->thresh_mult[THR_SR_NEAREST_NEARG] += 1200;
 
   /*
   rd->thresh_mult[THR_SR_NEAREST_NEWMV] += 1200;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEAREST_NEWL2] += 1200;
   rd->thresh_mult[THR_SR_NEAREST_NEWL3] += 1200;
   rd->thresh_mult[THR_SR_NEAREST_NEWB] += 1200;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEAREST_NEWA] += 1200;
   rd->thresh_mult[THR_SR_NEAREST_NEWG] += 1200;*/
 
   rd->thresh_mult[THR_SR_NEAR_NEWMV] += 1500;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEAR_NEWL2] += 1500;
   rd->thresh_mult[THR_SR_NEAR_NEWL3] += 1500;
   rd->thresh_mult[THR_SR_NEAR_NEWB] += 1500;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEAR_NEWA] += 1500;
   rd->thresh_mult[THR_SR_NEAR_NEWG] += 1500;
 
   rd->thresh_mult[THR_SR_ZERO_NEWMV] += 2000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_ZERO_NEWL2] += 2000;
   rd->thresh_mult[THR_SR_ZERO_NEWL3] += 2000;
   rd->thresh_mult[THR_SR_ZERO_NEWB] += 2000;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_ZERO_NEWA] += 2000;
   rd->thresh_mult[THR_SR_ZERO_NEWG] += 2000;
 
   rd->thresh_mult[THR_SR_NEW_NEWMV] += 1700;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEW_NEWL2] += 1700;
   rd->thresh_mult[THR_SR_NEW_NEWL3] += 1700;
   rd->thresh_mult[THR_SR_NEW_NEWB] += 1700;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_SR_NEW_NEWA] += 1700;
   rd->thresh_mult[THR_SR_NEW_NEWG] += 1700;
 #endif  // CONFIG_COMPOUND_SINGLEREF
 
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTLA] += 1000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2A] += 1000;
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3A] += 1000;
-#endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTGA] += 1000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTLB] += 1000;
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2B] += 1000;
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3B] += 1000;
@@ -1397,7 +1372,6 @@
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTLG] += 1000;
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTBA] += 1000;
 #endif  // CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 
   rd->thresh_mult[THR_COMP_NEAR_NEARLA] += 1200;
   rd->thresh_mult[THR_COMP_NEAREST_NEWLA] += 1500;
@@ -1407,7 +1381,6 @@
   rd->thresh_mult[THR_COMP_NEW_NEWLA] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROLA] += 2500;
 
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAR_NEARL2A] += 1200;
   rd->thresh_mult[THR_COMP_NEAREST_NEWL2A] += 1500;
   rd->thresh_mult[THR_COMP_NEW_NEARESTL2A] += 1500;
@@ -1423,7 +1396,6 @@
   rd->thresh_mult[THR_COMP_NEW_NEARL3A] += 1700;
   rd->thresh_mult[THR_COMP_NEW_NEWL3A] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROL3A] += 2500;
-#endif  // CONFIG_EXT_REFS
 
   rd->thresh_mult[THR_COMP_NEAR_NEARGA] += 1200;
   rd->thresh_mult[THR_COMP_NEAREST_NEWGA] += 1500;
@@ -1433,7 +1405,6 @@
   rd->thresh_mult[THR_COMP_NEW_NEWGA] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROGA] += 2500;
 
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAR_NEARLB] += 1200;
   rd->thresh_mult[THR_COMP_NEAREST_NEWLB] += 1500;
   rd->thresh_mult[THR_COMP_NEW_NEARESTLB] += 1500;
@@ -1531,7 +1502,6 @@
   rd->thresh_mult[THR_COMP_NEW_NEWBA] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROBA] += 2500;
 #endif  // CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 
   rd->thresh_mult[THR_H_PRED] += 2000;
   rd->thresh_mult[THR_V_PRED] += 2000;
@@ -1547,7 +1517,6 @@
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARL] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWL] += 2000;
 
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL2] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL2] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARL2] += 1500;
@@ -1557,14 +1526,12 @@
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL3] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARL3] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWL3] += 2000;
-#endif  // CONFIG_EXT_REFS
 
   rd->thresh_mult[THR_COMP_INTERINTRA_ZEROG] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTG] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARG] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWG] += 2000;
 
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_INTERINTRA_ZEROB] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTB] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARB] += 1500;
@@ -1574,7 +1541,6 @@
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTA2] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARA2] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWA2] += 2000;
-#endif  // CONFIG_EXT_REFS
 
   rd->thresh_mult[THR_COMP_INTERINTRA_ZEROA] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTA] += 1500;
@@ -1583,37 +1549,10 @@
 }
 
 void av1_set_rd_speed_thresholds_sub8x8(AV1_COMP *cpi) {
-  static const int thresh_mult[MAX_REFS] = {
-#if CONFIG_EXT_REFS
-    2500,
-    2500,
-    2500,
-    2500,
-    2500,
-    2500,
-    2500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    4500,
-    2500
-#else  // !CONFIG_EXT_REFS
-    2500,
-    2500,
-    2500,
-    4500,
-    4500,
-    2500
-#endif  // CONFIG_EXT_REFS
-  };
+  static const int thresh_mult[MAX_REFS] = { 2500, 2500, 2500, 2500, 2500,
+                                             2500, 2500, 4500, 4500, 4500,
+                                             4500, 4500, 4500, 4500, 4500,
+                                             4500, 4500, 4500, 4500, 2500 };
   RD_OPT *const rd = &cpi->rd;
   memcpy(rd->thresh_mult_sub8x8, thresh_mult, sizeof(thresh_mult));
 }
diff --git a/av1/encoder/rd.h b/av1/encoder/rd.h
index f57a8de..7651a00 100644
--- a/av1/encoder/rd.h
+++ b/av1/encoder/rd.h
@@ -50,102 +50,81 @@
 // const MODE_DEFINITION av1_mode_order[MAX_MODES] used in the rd code.
 typedef enum {
   THR_NEARESTMV,
-#if CONFIG_EXT_REFS
   THR_NEARESTL2,
   THR_NEARESTL3,
   THR_NEARESTB,
   THR_NEARESTA2,
-#endif  // CONFIG_EXT_REFS
   THR_NEARESTA,
   THR_NEARESTG,
 
   THR_DC,
 
   THR_NEWMV,
-#if CONFIG_EXT_REFS
   THR_NEWL2,
   THR_NEWL3,
   THR_NEWB,
   THR_NEWA2,
-#endif  // CONFIG_EXT_REFS
   THR_NEWA,
   THR_NEWG,
 
   THR_NEARMV,
-#if CONFIG_EXT_REFS
   THR_NEARL2,
   THR_NEARL3,
   THR_NEARB,
   THR_NEARA2,
-#endif  // CONFIG_EXT_REFS
   THR_NEARA,
   THR_NEARG,
 
   THR_ZEROMV,
-#if CONFIG_EXT_REFS
   THR_ZEROL2,
   THR_ZEROL3,
   THR_ZEROB,
   THR_ZEROA2,
-#endif  // CONFIG_EXT_REFS
   THR_ZEROA,
   THR_ZEROG,
 
 #if CONFIG_COMPOUND_SINGLEREF
   THR_SR_NEAREST_NEARMV,
-#if CONFIG_EXT_REFS
   THR_SR_NEAREST_NEARL2,
   THR_SR_NEAREST_NEARL3,
   THR_SR_NEAREST_NEARB,
-#endif  // CONFIG_EXT_REFS
   THR_SR_NEAREST_NEARG,
   THR_SR_NEAREST_NEARA,
 
   /*
   THR_SR_NEAREST_NEWMV,
-#if CONFIG_EXT_REFS
   THR_SR_NEAREST_NEWL2,
   THR_SR_NEAREST_NEWL3,
   THR_SR_NEAREST_NEWB,
-#endif  // CONFIG_EXT_REFS
   THR_SR_NEAREST_NEWG,
   THR_SR_NEAREST_NEWA,*/
 
   THR_SR_NEAR_NEWMV,
-#if CONFIG_EXT_REFS
   THR_SR_NEAR_NEWL2,
   THR_SR_NEAR_NEWL3,
   THR_SR_NEAR_NEWB,
-#endif  // CONFIG_EXT_REFS
   THR_SR_NEAR_NEWG,
   THR_SR_NEAR_NEWA,
 
   THR_SR_ZERO_NEWMV,
-#if CONFIG_EXT_REFS
   THR_SR_ZERO_NEWL2,
   THR_SR_ZERO_NEWL3,
   THR_SR_ZERO_NEWB,
-#endif  // CONFIG_EXT_REFS
   THR_SR_ZERO_NEWG,
   THR_SR_ZERO_NEWA,
 
   THR_SR_NEW_NEWMV,
-#if CONFIG_EXT_REFS
   THR_SR_NEW_NEWL2,
   THR_SR_NEW_NEWL3,
   THR_SR_NEW_NEWB,
-#endif  // CONFIG_EXT_REFS
   THR_SR_NEW_NEWG,
   THR_SR_NEW_NEWA,
 #endif  // CONFIG_COMPOUND_SINGLEREF
 
   THR_COMP_NEAREST_NEARESTLA,
-#if CONFIG_EXT_REFS
   THR_COMP_NEAREST_NEARESTL2A,
   THR_COMP_NEAREST_NEARESTL3A,
-#endif  // CONFIG_EXT_REFS
   THR_COMP_NEAREST_NEARESTGA,
-#if CONFIG_EXT_REFS
   THR_COMP_NEAREST_NEARESTLB,
   THR_COMP_NEAREST_NEARESTL2B,
   THR_COMP_NEAREST_NEARESTL3B,
@@ -160,7 +139,6 @@
   THR_COMP_NEAREST_NEARESTLG,
   THR_COMP_NEAREST_NEARESTBA,
 #endif  // CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 
   THR_PAETH,
 
@@ -178,7 +156,6 @@
   THR_COMP_NEW_NEWLA,
   THR_COMP_ZERO_ZEROLA,
 
-#if CONFIG_EXT_REFS
   THR_COMP_NEAR_NEARL2A,
   THR_COMP_NEW_NEARESTL2A,
   THR_COMP_NEAREST_NEWL2A,
@@ -194,7 +171,6 @@
   THR_COMP_NEAR_NEWL3A,
   THR_COMP_NEW_NEWL3A,
   THR_COMP_ZERO_ZEROL3A,
-#endif  // CONFIG_EXT_REFS
 
   THR_COMP_NEAR_NEARGA,
   THR_COMP_NEW_NEARESTGA,
@@ -204,7 +180,6 @@
   THR_COMP_NEW_NEWGA,
   THR_COMP_ZERO_ZEROGA,
 
-#if CONFIG_EXT_REFS
   THR_COMP_NEAR_NEARLB,
   THR_COMP_NEW_NEARESTLB,
   THR_COMP_NEAREST_NEWLB,
@@ -302,7 +277,6 @@
   THR_COMP_NEW_NEWBA,
   THR_COMP_ZERO_ZEROBA,
 #endif  // CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 
   THR_H_PRED,
   THR_V_PRED,
@@ -318,7 +292,6 @@
   THR_COMP_INTERINTRA_NEARL,
   THR_COMP_INTERINTRA_NEWL,
 
-#if CONFIG_EXT_REFS
   THR_COMP_INTERINTRA_ZEROL2,
   THR_COMP_INTERINTRA_NEARESTL2,
   THR_COMP_INTERINTRA_NEARL2,
@@ -328,14 +301,12 @@
   THR_COMP_INTERINTRA_NEARESTL3,
   THR_COMP_INTERINTRA_NEARL3,
   THR_COMP_INTERINTRA_NEWL3,
-#endif  // CONFIG_EXT_REFS
 
   THR_COMP_INTERINTRA_ZEROG,
   THR_COMP_INTERINTRA_NEARESTG,
   THR_COMP_INTERINTRA_NEARG,
   THR_COMP_INTERINTRA_NEWG,
 
-#if CONFIG_EXT_REFS
   THR_COMP_INTERINTRA_ZEROB,
   THR_COMP_INTERINTRA_NEARESTB,
   THR_COMP_INTERINTRA_NEARB,
@@ -345,7 +316,6 @@
   THR_COMP_INTERINTRA_NEARESTA2,
   THR_COMP_INTERINTRA_NEARA2,
   THR_COMP_INTERINTRA_NEWA2,
-#endif  // CONFIG_EXT_REFS
 
   THR_COMP_INTERINTRA_ZEROA,
   THR_COMP_INTERINTRA_NEARESTA,
@@ -356,23 +326,18 @@
 
 typedef enum {
   THR_LAST,
-#if CONFIG_EXT_REFS
   THR_LAST2,
   THR_LAST3,
   THR_BWDR,
   THR_ALTR2,
-#endif  // CONFIG_EXT_REFS
   THR_GOLD,
   THR_ALTR,
 
   THR_COMP_LA,
-#if CONFIG_EXT_REFS
   THR_COMP_L2A,
   THR_COMP_L3A,
-#endif  // CONFIG_EXT_REFS
   THR_COMP_GA,
 
-#if CONFIG_EXT_REFS
   THR_COMP_LB,
   THR_COMP_L2B,
   THR_COMP_L3B,
@@ -382,7 +347,6 @@
   THR_COMP_L2A2,
   THR_COMP_L3A2,
   THR_COMP_GA2,
-#endif  // CONFIG_EXT_REFS
 
   THR_INTRA,
 
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index f1e3f16..639803d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -80,8 +80,6 @@
 #endif  // USE_EXTRA_FILTER
 #endif  // CONFIG_DUAL_FILTER
 
-#if CONFIG_EXT_REFS
-
 #define LAST_FRAME_MODE_MASK                                          \
   ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |     \
    (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
@@ -111,18 +109,6 @@
    (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | \
    (1 << ALTREF2_FRAME))
 
-#else  // !CONFIG_EXT_REFS
-
-#define LAST_FRAME_MODE_MASK \
-  ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
-#define GOLDEN_FRAME_MODE_MASK \
-  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
-#define ALTREF_FRAME_MODE_MASK \
-  ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | (1 << INTRA_FRAME))
-
-#endif  // CONFIG_EXT_REFS
-
-#if CONFIG_EXT_REFS
 #if CONFIG_EXT_COMP_REFS
 #define SECOND_REF_FRAME_MASK                                         \
   ((1 << ALTREF_FRAME) | (1 << ALTREF2_FRAME) | (1 << BWDREF_FRAME) | \
@@ -131,9 +117,6 @@
 #define SECOND_REF_FRAME_MASK \
   ((1 << ALTREF_FRAME) | (1 << ALTREF2_FRAME) | (1 << BWDREF_FRAME) | 0x01)
 #endif  // CONFIG_EXT_COMP_REFS
-#else   // !CONFIG_EXT_REFS
-#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | 0x01)
-#endif  // CONFIG_EXT_REFS
 
 #define MIN_EARLY_TERM_INDEX 3
 #define NEW_MV_DISCOUNT_FACTOR 8
@@ -179,44 +162,36 @@
 #define LAST_NEW_MV_INDEX 6
 static const MODE_DEFINITION av1_mode_order[MAX_MODES] = {
   { NEARESTMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { NEARESTMV, { LAST2_FRAME, NONE_FRAME } },
   { NEARESTMV, { LAST3_FRAME, NONE_FRAME } },
   { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } },
   { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } },
   { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } },
 
   { DC_PRED, { INTRA_FRAME, NONE_FRAME } },
 
   { NEWMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { NEWMV, { LAST2_FRAME, NONE_FRAME } },
   { NEWMV, { LAST3_FRAME, NONE_FRAME } },
   { NEWMV, { BWDREF_FRAME, NONE_FRAME } },
   { NEWMV, { ALTREF2_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { NEWMV, { ALTREF_FRAME, NONE_FRAME } },
   { NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
 
   { NEARMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { NEARMV, { LAST2_FRAME, NONE_FRAME } },
   { NEARMV, { LAST3_FRAME, NONE_FRAME } },
   { NEARMV, { BWDREF_FRAME, NONE_FRAME } },
   { NEARMV, { ALTREF2_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { NEARMV, { ALTREF_FRAME, NONE_FRAME } },
   { NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
 
   { ZEROMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { ZEROMV, { LAST2_FRAME, NONE_FRAME } },
   { ZEROMV, { LAST3_FRAME, NONE_FRAME } },
   { ZEROMV, { BWDREF_FRAME, NONE_FRAME } },
   { ZEROMV, { ALTREF2_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { ZEROMV, { GOLDEN_FRAME, NONE_FRAME } },
   { ZEROMV, { ALTREF_FRAME, NONE_FRAME } },
 
@@ -225,59 +200,46 @@
 #if CONFIG_COMPOUND_SINGLEREF
   // Single ref comp mode
   { SR_NEAREST_NEARMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { SR_NEAREST_NEARMV, { LAST2_FRAME, NONE_FRAME } },
   { SR_NEAREST_NEARMV, { LAST3_FRAME, NONE_FRAME } },
   { SR_NEAREST_NEARMV, { BWDREF_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { SR_NEAREST_NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
   { SR_NEAREST_NEARMV, { ALTREF_FRAME, NONE_FRAME } },
 
   /*
   { SR_NEAREST_NEWMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { SR_NEAREST_NEWMV, { LAST2_FRAME, NONE_FRAME } },
   { SR_NEAREST_NEWMV, { LAST3_FRAME, NONE_FRAME } },
   { SR_NEAREST_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { SR_NEAREST_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
   { SR_NEAREST_NEWMV, { ALTREF_FRAME, NONE_FRAME } },*/
 
   { SR_NEAR_NEWMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { SR_NEAR_NEWMV, { LAST2_FRAME, NONE_FRAME } },
   { SR_NEAR_NEWMV, { LAST3_FRAME, NONE_FRAME } },
   { SR_NEAR_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { SR_NEAR_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
   { SR_NEAR_NEWMV, { ALTREF_FRAME, NONE_FRAME } },
 
   { SR_ZERO_NEWMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { SR_ZERO_NEWMV, { LAST2_FRAME, NONE_FRAME } },
   { SR_ZERO_NEWMV, { LAST3_FRAME, NONE_FRAME } },
   { SR_ZERO_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { SR_ZERO_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
   { SR_ZERO_NEWMV, { ALTREF_FRAME, NONE_FRAME } },
 
   { SR_NEW_NEWMV, { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
   { SR_NEW_NEWMV, { LAST2_FRAME, NONE_FRAME } },
   { SR_NEW_NEWMV, { LAST3_FRAME, NONE_FRAME } },
   { SR_NEW_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { SR_NEW_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
   { SR_NEW_NEWMV, { ALTREF_FRAME, NONE_FRAME } },
 #endif  // CONFIG_COMPOUND_SINGLEREF
 
   { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
-#if CONFIG_EXT_REFS
   { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
   { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
-#endif  // CONFIG_EXT_REFS
   { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
-#if CONFIG_EXT_REFS
   { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
   { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
   { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
@@ -293,7 +255,6 @@
   { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
   { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
 #endif  // CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 
   { PAETH_PRED, { INTRA_FRAME, NONE_FRAME } },
 
@@ -311,7 +272,6 @@
   { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
   { ZERO_ZEROMV, { LAST_FRAME, ALTREF_FRAME } },
 
-#if CONFIG_EXT_REFS
   { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
   { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
   { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
@@ -327,7 +287,6 @@
   { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
   { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
   { ZERO_ZEROMV, { LAST3_FRAME, ALTREF_FRAME } },
-#endif  // CONFIG_EXT_REFS
 
   { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
   { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
@@ -337,7 +296,6 @@
   { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
   { ZERO_ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } },
 
-#if CONFIG_EXT_REFS
   { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
   { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
   { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
@@ -435,7 +393,6 @@
   { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
   { ZERO_ZEROMV, { BWDREF_FRAME, ALTREF_FRAME } },
 #endif  // CONFIG_EXT_COMP_REFS
-#endif  // CONFIG_EXT_REFS
 
   { H_PRED, { INTRA_FRAME, NONE_FRAME } },
   { V_PRED, { INTRA_FRAME, NONE_FRAME } },
@@ -451,7 +408,6 @@
   { NEARMV, { LAST_FRAME, INTRA_FRAME } },
   { NEWMV, { LAST_FRAME, INTRA_FRAME } },
 
-#if CONFIG_EXT_REFS
   { ZEROMV, { LAST2_FRAME, INTRA_FRAME } },
   { NEARESTMV, { LAST2_FRAME, INTRA_FRAME } },
   { NEARMV, { LAST2_FRAME, INTRA_FRAME } },
@@ -461,14 +417,12 @@
   { NEARESTMV, { LAST3_FRAME, INTRA_FRAME } },
   { NEARMV, { LAST3_FRAME, INTRA_FRAME } },
   { NEWMV, { LAST3_FRAME, INTRA_FRAME } },
-#endif  // CONFIG_EXT_REFS
 
   { ZEROMV, { GOLDEN_FRAME, INTRA_FRAME } },
   { NEARESTMV, { GOLDEN_FRAME, INTRA_FRAME } },
   { NEARMV, { GOLDEN_FRAME, INTRA_FRAME } },
   { NEWMV, { GOLDEN_FRAME, INTRA_FRAME } },
 
-#if CONFIG_EXT_REFS
   { ZEROMV, { BWDREF_FRAME, INTRA_FRAME } },
   { NEARESTMV, { BWDREF_FRAME, INTRA_FRAME } },
   { NEARMV, { BWDREF_FRAME, INTRA_FRAME } },
@@ -478,7 +432,6 @@
   { NEARESTMV, { ALTREF2_FRAME, INTRA_FRAME } },
   { NEARMV, { ALTREF2_FRAME, INTRA_FRAME } },
   { NEWMV, { ALTREF2_FRAME, INTRA_FRAME } },
-#endif  // CONFIG_EXT_REFS
 
   { ZEROMV, { ALTREF_FRAME, INTRA_FRAME } },
   { NEARESTMV, { ALTREF_FRAME, INTRA_FRAME } },
@@ -6955,24 +6908,18 @@
     if (cm->reference_mode != COMPOUND_REFERENCE) {
       aom_prob ref_single_p1 = av1_get_pred_prob_single_ref_p1(cm, xd);
       aom_prob ref_single_p2 = av1_get_pred_prob_single_ref_p2(cm, xd);
-#if CONFIG_EXT_REFS
       aom_prob ref_single_p3 = av1_get_pred_prob_single_ref_p3(cm, xd);
       aom_prob ref_single_p4 = av1_get_pred_prob_single_ref_p4(cm, xd);
       aom_prob ref_single_p5 = av1_get_pred_prob_single_ref_p5(cm, xd);
       aom_prob ref_single_p6 = av1_get_pred_prob_single_ref_p6(cm, xd);
-#endif  // CONFIG_EXT_REFS
 
       unsigned int base_cost = av1_cost_bit(intra_inter_p, 1);
 
-      ref_costs_single[LAST_FRAME] =
-#if CONFIG_EXT_REFS
-          ref_costs_single[LAST2_FRAME] = ref_costs_single[LAST3_FRAME] =
-              ref_costs_single[BWDREF_FRAME] = ref_costs_single[ALTREF2_FRAME] =
-#endif  // CONFIG_EXT_REFS
-                  ref_costs_single[GOLDEN_FRAME] =
-                      ref_costs_single[ALTREF_FRAME] = base_cost;
+      ref_costs_single[LAST_FRAME] = ref_costs_single[LAST2_FRAME] =
+          ref_costs_single[LAST3_FRAME] = ref_costs_single[BWDREF_FRAME] =
+              ref_costs_single[ALTREF2_FRAME] = ref_costs_single[GOLDEN_FRAME] =
+                  ref_costs_single[ALTREF_FRAME] = base_cost;
 
-#if CONFIG_EXT_REFS
       ref_costs_single[LAST_FRAME] += av1_cost_bit(ref_single_p1, 0);
       ref_costs_single[LAST2_FRAME] += av1_cost_bit(ref_single_p1, 0);
       ref_costs_single[LAST3_FRAME] += av1_cost_bit(ref_single_p1, 0);
@@ -6998,34 +6945,22 @@
 
       ref_costs_single[BWDREF_FRAME] += av1_cost_bit(ref_single_p6, 0);
       ref_costs_single[ALTREF2_FRAME] += av1_cost_bit(ref_single_p6, 1);
-#else   // !CONFIG_EXT_REFS
-      ref_costs_single[LAST_FRAME] += av1_cost_bit(ref_single_p1, 0);
-      ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p1, 1);
-      ref_costs_single[ALTREF_FRAME] += av1_cost_bit(ref_single_p1, 1);
-
-      ref_costs_single[GOLDEN_FRAME] += av1_cost_bit(ref_single_p2, 0);
-      ref_costs_single[ALTREF_FRAME] += av1_cost_bit(ref_single_p2, 1);
-#endif  // CONFIG_EXT_REFS
     } else {
       ref_costs_single[LAST_FRAME] = 512;
-#if CONFIG_EXT_REFS
       ref_costs_single[LAST2_FRAME] = 512;
       ref_costs_single[LAST3_FRAME] = 512;
       ref_costs_single[BWDREF_FRAME] = 512;
       ref_costs_single[ALTREF2_FRAME] = 512;
-#endif  // CONFIG_EXT_REFS
       ref_costs_single[GOLDEN_FRAME] = 512;
       ref_costs_single[ALTREF_FRAME] = 512;
     }
 
     if (cm->reference_mode != SINGLE_REFERENCE) {
       aom_prob ref_comp_p = av1_get_pred_prob_comp_ref_p(cm, xd);
-#if CONFIG_EXT_REFS
       aom_prob ref_comp_p1 = av1_get_pred_prob_comp_ref_p1(cm, xd);
       aom_prob ref_comp_p2 = av1_get_pred_prob_comp_ref_p2(cm, xd);
       aom_prob bwdref_comp_p = av1_get_pred_prob_comp_bwdref_p(cm, xd);
       aom_prob bwdref_comp_p1 = av1_get_pred_prob_comp_bwdref_p1(cm, xd);
-#endif  // CONFIG_EXT_REFS
 
       unsigned int base_cost = av1_cost_bit(intra_inter_p, 1);
 
@@ -7089,20 +7024,15 @@
           base_cost + av1_cost_bit(comp_ref_type_p, 0) +
           av1_cost_bit(uni_comp_ref_p, 1);
 
-#else  // !CONFIG_EXT_COMP_REFS
+#else   // !CONFIG_EXT_COMP_REFS
 
-      ref_costs_comp[LAST_FRAME] =
-#if CONFIG_EXT_REFS
-          ref_costs_comp[LAST2_FRAME] = ref_costs_comp[LAST3_FRAME] =
-#endif  // CONFIG_EXT_REFS
-              ref_costs_comp[GOLDEN_FRAME] = base_cost;
+      ref_costs_comp[LAST_FRAME] = ref_costs_comp[LAST2_FRAME] =
+          ref_costs_comp[LAST3_FRAME] = ref_costs_comp[GOLDEN_FRAME] =
+              base_cost;
 
-#if CONFIG_EXT_REFS
       ref_costs_comp[BWDREF_FRAME] = ref_costs_comp[ALTREF2_FRAME] =
           ref_costs_comp[ALTREF_FRAME] = 0;
-#endif  // CONFIG_EXT_REFS
 
-#if CONFIG_EXT_REFS
       ref_costs_comp[LAST_FRAME] += av1_cost_bit(ref_comp_p, 0);
       ref_costs_comp[LAST2_FRAME] += av1_cost_bit(ref_comp_p, 0);
       ref_costs_comp[LAST3_FRAME] += av1_cost_bit(ref_comp_p, 1);
@@ -7122,10 +7052,6 @@
 
       ref_costs_comp[BWDREF_FRAME] += av1_cost_bit(bwdref_comp_p1, 0);
       ref_costs_comp[ALTREF2_FRAME] += av1_cost_bit(bwdref_comp_p1, 1);
-#else   // !CONFIG_EXT_REFS
-      ref_costs_comp[LAST_FRAME] += av1_cost_bit(ref_comp_p, 0);
-      ref_costs_comp[GOLDEN_FRAME] += av1_cost_bit(ref_comp_p, 1);
-#endif  // CONFIG_EXT_REFS
 #endif  // CONFIG_EXT_COMP_REFS
     } else {
 #if CONFIG_EXT_COMP_REFS
@@ -7138,15 +7064,13 @@
       ref_costs_comp[LAST_FRAME][LAST3_FRAME] = 512;
       ref_costs_comp[LAST_FRAME][GOLDEN_FRAME] = 512;
       ref_costs_comp[BWDREF_FRAME][ALTREF_FRAME] = 512;
-#else  // !CONFIG_EXT_COMP_REFS
+#else   // !CONFIG_EXT_COMP_REFS
       ref_costs_comp[LAST_FRAME] = 512;
-#if CONFIG_EXT_REFS
       ref_costs_comp[LAST2_FRAME] = 512;
       ref_costs_comp[LAST3_FRAME] = 512;
       ref_costs_comp[BWDREF_FRAME] = 512;
       ref_costs_comp[ALTREF2_FRAME] = 512;
       ref_costs_comp[ALTREF_FRAME] = 512;
-#endif  // CONFIG_EXT_REFS
       ref_costs_comp[GOLDEN_FRAME] = 512;
 #endif  // CONFIG_EXT_COMP_REFS
     }
@@ -10330,20 +10254,14 @@
   int_mv single_newmv[TOTAL_REFS_PER_FRAME] = { { 0 } };
   int single_newmv_rate[TOTAL_REFS_PER_FRAME] = { 0 };
   int64_t modelled_rd[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME];
-  static const int flag_list[TOTAL_REFS_PER_FRAME] = {
-    0,
-    AOM_LAST_FLAG,
-#if CONFIG_EXT_REFS
-    AOM_LAST2_FLAG,
-    AOM_LAST3_FLAG,
-#endif  // CONFIG_EXT_REFS
-    AOM_GOLD_FLAG,
-#if CONFIG_EXT_REFS
-    AOM_BWD_FLAG,
-    AOM_ALT2_FLAG,
-#endif  // CONFIG_EXT_REFS
-    AOM_ALT_FLAG
-  };
+  static const int flag_list[TOTAL_REFS_PER_FRAME] = { 0,
+                                                       AOM_LAST_FLAG,
+                                                       AOM_LAST2_FLAG,
+                                                       AOM_LAST3_FLAG,
+                                                       AOM_GOLD_FLAG,
+                                                       AOM_BWD_FLAG,
+                                                       AOM_ALT2_FLAG,
+                                                       AOM_ALT_FLAG };
   int64_t best_rd = best_rd_so_far;
   int best_rate_y = INT_MAX, best_rate_uv = INT_MAX;
   int64_t best_pred_diff[REFERENCE_MODES];
@@ -10595,12 +10513,9 @@
     // because they may result in zero-zero MVs but be cheaper.
     if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) {
       int_mv zeromv;
-      ref_frame_skip_mask[0] = (1 << LAST_FRAME) |
-#if CONFIG_EXT_REFS
-                               (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |
-                               (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) |
-#endif  // CONFIG_EXT_REFS
-                               (1 << GOLDEN_FRAME);
+      ref_frame_skip_mask[0] = (1 << LAST_FRAME) | (1 << LAST2_FRAME) |
+                               (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) |
+                               (1 << ALTREF2_FRAME) | (1 << GOLDEN_FRAME);
       ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;
       // TODO(zoeliu): To further explore whether following needs to be done for
       //               BWDREF_FRAME as well.
@@ -10748,7 +10663,6 @@
           ref_frame_skip_mask[0] |= LAST_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-#if CONFIG_EXT_REFS
         case LAST2_FRAME:
           ref_frame_skip_mask[0] |= LAST2_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
@@ -10757,12 +10671,10 @@
           ref_frame_skip_mask[0] |= LAST3_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-#endif  // CONFIG_EXT_REFS
         case GOLDEN_FRAME:
           ref_frame_skip_mask[0] |= GOLDEN_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-#if CONFIG_EXT_REFS
         case BWDREF_FRAME:
           ref_frame_skip_mask[0] |= BWDREF_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
@@ -10771,11 +10683,9 @@
           ref_frame_skip_mask[0] |= ALTREF2_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-#endif  // CONFIG_EXT_REFS
-        case ALTREF_FRAME: ref_frame_skip_mask[0] |= ALTREF_FRAME_MODE_MASK;
-#if CONFIG_EXT_REFS
+        case ALTREF_FRAME:
+          ref_frame_skip_mask[0] |= ALTREF_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
-#endif  // CONFIG_EXT_REFS
           break;
         case NONE_FRAME:
         case TOTAL_REFS_PER_FRAME:
@@ -10828,7 +10738,6 @@
     if (cpi->oxcf.motion_vector_unit_test && ref_frame == INTRA_FRAME) continue;
 
 #if CONFIG_ONE_SIDED_COMPOUND && !CONFIG_EXT_COMP_REFS  // Changes LL bitstream
-#if CONFIG_EXT_REFS
     if (cpi->oxcf.pass == 0) {
       // Complexity-compression trade-offs
       // if (ref_frame == ALTREF_FRAME) continue;
@@ -10836,7 +10745,6 @@
       if (second_ref_frame == ALTREF_FRAME) continue;
       // if (second_ref_frame == BWDREF_FRAME) continue;
     }
-#endif  // CONFIG_EXT_REFS
 #endif  // CONFIG_ONE_SIDED_COMPOUND && !CONFIG_EXT_COMP_REFS
     comp_pred = second_ref_frame > INTRA_FRAME;
     if (comp_pred) {
@@ -11450,11 +11358,9 @@
     if (comp_pred) {
 #if CONFIG_EXT_COMP_REFS
       rate2 += ref_costs_comp[ref_frame][second_ref_frame];
-#else  // !CONFIG_EXT_COMP_REFS
+#else   // !CONFIG_EXT_COMP_REFS
       rate2 += ref_costs_comp[ref_frame];
-#if CONFIG_EXT_REFS
       rate2 += ref_costs_comp[second_ref_frame];
-#endif  // CONFIG_EXT_REFS
 #endif  // CONFIG_EXT_COMP_REFS
     } else {
       rate2 += ref_costs_single[ref_frame];
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 5608d03..4e7c8df 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -328,8 +328,7 @@
   RD_OPT *const rd = &cpi->rd;
   int i;
 
-// Limit memory usage for high resolutions
-#if CONFIG_EXT_REFS
+  // Limit memory usage for high resolutions
   // TODO(zoeliu): Temporary solution to resolve the insufficient RAM issue for
   //               ext-refs. Need to work with @yunqingwang to have a more
   //               effective solution.
@@ -340,14 +339,6 @@
              (oxcf->profile != PROFILE_0)) {
     sf->use_upsampled_references = 0;
   }
-#else
-  if (AOMMIN(cm->width, cm->height) > 1080) {
-    sf->use_upsampled_references = 0;
-  } else if ((AOMMIN(cm->width, cm->height) > 720) &&
-             (oxcf->profile != PROFILE_0)) {
-    sf->use_upsampled_references = 0;
-  }
-#endif  // CONFIG_EXT_REFS
 
   if (oxcf->mode == GOOD) {
     set_good_speed_feature_framesize_dependent(cpi, sf, oxcf->speed);
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index daa6476..61fd75d 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -647,17 +647,14 @@
   int frames_to_blur_forward;
   struct scale_factors sf;
   YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS] = { NULL };
-#if CONFIG_EXT_REFS
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
-#endif  // CONFIG_EXT_REFS
 
   // Apply context specific adjustments to the arnr filter parameters.
   adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, &strength);
-// TODO(weitinglin): Currently, we enforce the filtering strength on
-//                   extra ARFs' to be zeros. We should investigate in which
-//                   case it is more beneficial to use non-zero strength
-//                   filtering.
-#if CONFIG_EXT_REFS
+  // TODO(weitinglin): Currently, we enforce the filtering strength on
+  //                   extra ARFs' to be zeros. We should investigate in which
+  //                   case it is more beneficial to use non-zero strength
+  //                   filtering.
   if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) {
     strength = 0;
     frames_to_blur = 1;
@@ -685,7 +682,6 @@
     cpi->is_arf_filter_off[which_arf] = 1;
   else
     cpi->is_arf_filter_off[which_arf] = 0;
-#endif  // CONFIG_EXT_REFS
 
   frames_to_blur_backward = (frames_to_blur / 2);
   frames_to_blur_forward = ((frames_to_blur - 1) / 2);