Merge bi-predictive frames to EXT_REFS

This patch removed the experiment of BIDIR_PRED and merged the feature
into the experiment of EXT_REFS:

(1) Each frame now has up to 6 reference frames, namely
    LAST_FRAME, LAST2_FRAME, LAST3_FRAME, GOLDEN_FRAME, (forward) and
    BWDREF_FRAME, ALTREF_FRAME (backward);
    LAST4_FRAME has been removed;
(2) First pass still keeps the 8 updates:
    KF_UPDATE, LF_UPDATE, GF_UPDATE, ARF_UPDATE, OVERLAY_UPDATE, and
    BRF_UPDATE, LAST_BIPRED_UPDATE, BI_PRED_UPDATE;
(3) show_existing_frame==1 is supported in the experiment of EXT_REFS;
(4) New encoding modes are added for both single-ref and compound cases,
    through the use of the 2 extra forward references (LAST2 & LAST3)
    and the 1 extra backward reference (BWDREF).

RD performance wise, using Overall PSNR: Avg/BDRate
        Bipred only      Prev EXT_REFS    Current EXT_REFS with bipred
lowres: -3.474/-3.324    -1.748/-1.586    -4.613/-4.387
derflr: -2.097/-1.353    -1.439/-1.215    -3.120/-2.252
midres: -2.129/-1.901    -1.345/-1.185    -2.898/-2.636

If in vp10/encoder/firstpass.h, change BFG_INTERVAL from 2 to 3, i.e. to
use 2 bi-predictive frames than 1, a further improvement may be
obtained:
                 Current EXT_REFS with bipred
        1 bi-predictive frame    2 bi-predictive frames
lowres: -4.613/-4.387            -4.675/-4.465
derflr: -3.120/-2.252            -3.333/-2.516
midres: -2.898/-2.636            -3.406/-3.095

Change-Id: Ib06fe9ea0a5cfd7418a1d79b978ee9d80bf191cb
diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index d3ab207..801cabb 100644
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -102,29 +102,18 @@
 
 #define LAST2_FRAME     2
 #define LAST3_FRAME     3
-#define LAST4_FRAME     4
-#define GOLDEN_FRAME    5
+#define GOLDEN_FRAME    4
+#define BWDREF_FRAME    5
 #define ALTREF_FRAME    6
 #define MAX_REF_FRAMES  7
-#define LAST_REF_FRAMES (LAST4_FRAME - LAST_FRAME + 1)
+#define LAST_REF_FRAMES (LAST3_FRAME - LAST_FRAME + 1)
 
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
-
-#define GOLDEN_FRAME    2
-#define BWDREF_FRAME    3
-#define ALTREF_FRAME    4
-#define MAX_REF_FRAMES  5
-
-#else  // CONFIG_BIDIR_PRED
+#else
 
 #define GOLDEN_FRAME    2
 #define ALTREF_FRAME    3
 #define MAX_REF_FRAMES  4
 
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
 
 typedef int8_t MV_REFERENCE_FRAME;
diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index 63ef9cf..8fb88b2 100644
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -428,32 +428,21 @@
 
 
 #if CONFIG_EXT_REFS
-static const vpx_prob default_comp_ref_p[REF_CONTEXTS][COMP_REFS - 1] = {
-  // TODO(zoeliu): To adjust the initial prob values.
-  {  33,  16,  16,  16 },
-  {  77,  74,  74,  74 },
-  { 142, 142, 142, 142 },
-  { 172, 170, 170, 170 },
-  { 238, 247, 247, 247 }
-};
-
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
-// TODO(zoeliu): To adjust the initial prob values.
 static const vpx_prob default_comp_ref_p[REF_CONTEXTS][FWD_REFS - 1] = {
-//  { 50 }, { 126 }, { 123 }, { 221 }, { 226 }
-  { 33 }, { 77 }, { 142 }, { 172 }, { 238 }
+  // TODO(zoeliu): To adjust the initial prob values.
+  {  33,  16,  16 },
+  {  77,  74,  74 },
+  { 142, 142, 142 },
+  { 172, 170, 170 },
+  { 238, 247, 247 }
 };
 static const vpx_prob default_comp_bwdref_p[REF_CONTEXTS][BWD_REFS - 1] = {
   { 16 }, { 74 }, { 142 }, { 170 }, { 247 }
 };
-#else  // CONFIG_BIDIR_PRED
+#else
 static const vpx_prob default_comp_ref_p[REF_CONTEXTS][COMP_REFS - 1] = {
   { 50 }, { 126 }, { 123 }, { 221 }, { 226 }
 };
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
 
 static const vpx_prob default_single_ref_p[REF_CONTEXTS][SINGLE_REFS - 1] = {
@@ -463,20 +452,12 @@
   { 142, 142, 142, 142, 142 },
   { 172, 170, 170, 170, 170 },
   { 238, 247, 247, 247, 247 }
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  {  33,  16,  16 },
-  {  77,  74,  74 },
-  { 142, 142, 142 },
-  { 172, 170, 170 },
-  { 238, 247, 247 }
-#else  // CONFIG_BIDIR_PRED
+#else
   {  33,  16 },
   {  77,  74 },
   { 142, 142 },
   { 172, 170 },
   { 238, 247 }
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 };
 
@@ -1311,9 +1292,9 @@
   vp10_copy(fc->intra_inter_prob, default_intra_inter_p);
   vp10_copy(fc->comp_inter_prob, default_comp_inter_p);
   vp10_copy(fc->comp_ref_prob, default_comp_ref_p);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   vp10_copy(fc->comp_bwdref_prob, default_comp_bwdref_p);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   vp10_copy(fc->single_ref_prob, default_single_ref_p);
   vp10_copy(fc->tx_size_probs, default_tx_size_prob);
 #if CONFIG_VAR_TX
@@ -1382,7 +1363,7 @@
     fc->comp_inter_prob[i] = vp10_mode_mv_merge_probs(
         pre_fc->comp_inter_prob[i], counts->comp_inter[i]);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   for (i = 0; i < REF_CONTEXTS; i++)
     for (j = 0; j < (FWD_REFS - 1); j++)
       fc->comp_ref_prob[i][j] = mode_mv_merge_probs(
@@ -1396,7 +1377,7 @@
     for (j = 0; j < (COMP_REFS - 1); j++)
       fc->comp_ref_prob[i][j] = mode_mv_merge_probs(
           pre_fc->comp_ref_prob[i][j], counts->comp_ref[i][j]);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   for (i = 0; i < REF_CONTEXTS; i++)
     for (j = 0; j < (SINGLE_REFS - 1); j++)
@@ -1599,7 +1580,7 @@
 #if CONFIG_EXT_REFS
   lf->ref_deltas[LAST2_FRAME] = lf->ref_deltas[LAST_FRAME];
   lf->ref_deltas[LAST3_FRAME] = lf->ref_deltas[LAST_FRAME];
-  lf->ref_deltas[LAST4_FRAME] = lf->ref_deltas[LAST_FRAME];
+  lf->ref_deltas[BWDREF_FRAME] = lf->ref_deltas[LAST_FRAME];
 #endif  // CONFIG_EXT_REFS
   lf->ref_deltas[GOLDEN_FRAME] = -1;
   lf->ref_deltas[ALTREF_FRAME] = -1;
diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h
index 1c9f869..71e79d9 100644
--- a/vp10/common/entropymode.h
+++ b/vp10/common/entropymode.h
@@ -85,12 +85,12 @@
   vpx_prob intra_inter_prob[INTRA_INTER_CONTEXTS];
   vpx_prob comp_inter_prob[COMP_INTER_CONTEXTS];
   vpx_prob single_ref_prob[REF_CONTEXTS][SINGLE_REFS-1];
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   vpx_prob comp_ref_prob[REF_CONTEXTS][FWD_REFS-1];
   vpx_prob comp_bwdref_prob[REF_CONTEXTS][BWD_REFS-1];
 #else
   vpx_prob comp_ref_prob[REF_CONTEXTS][COMP_REFS-1];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   vpx_prob tx_size_probs[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES - 1];
 #if CONFIG_VAR_TX
   vpx_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
@@ -160,12 +160,12 @@
   unsigned int intra_inter[INTRA_INTER_CONTEXTS][2];
   unsigned int comp_inter[COMP_INTER_CONTEXTS][2];
   unsigned int single_ref[REF_CONTEXTS][SINGLE_REFS-1][2];
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   unsigned int comp_ref[REF_CONTEXTS][FWD_REFS-1][2];
   unsigned int comp_bwdref[REF_CONTEXTS][BWD_REFS-1][2];
 #else
   unsigned int comp_ref[REF_CONTEXTS][COMP_REFS-1][2];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   unsigned int tx_size_totals[TX_SIZES];
   unsigned int tx_size[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
 #if CONFIG_VAR_TX
diff --git a/vp10/common/enums.h b/vp10/common/enums.h
index c49e8c9..d75b0f7 100644
--- a/vp10/common/enums.h
+++ b/vp10/common/enums.h
@@ -204,21 +204,14 @@
 #if CONFIG_EXT_REFS
   VP9_LAST2_FLAG = 1 << 1,
   VP9_LAST3_FLAG = 1 << 2,
-  VP9_LAST4_FLAG = 1 << 3,
-  VP9_GOLD_FLAG = 1 << 4,
+  VP9_GOLD_FLAG = 1 << 3,
+  VP9_BWD_FLAG = 1 << 4,
   VP9_ALT_FLAG = 1 << 5,
   VP9_REFFRAME_ALL = (1 << 6) - 1
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  VP9_GOLD_FLAG = 1 << 1,
-  VP9_BWD_FLAG = 1 << 2,
-  VP9_ALT_FLAG = 1 << 3,
-  VP9_REFFRAME_ALL = (1 << 4) - 1
-#else  // CONFIG_BIDIR_PRED
+#else
   VP9_GOLD_FLAG = 1 << 1,
   VP9_ALT_FLAG = 1 << 2,
   VP9_REFFRAME_ALL = (1 << 3) - 1
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 } VP9_REFFRAME;
 
@@ -389,24 +382,14 @@
 #endif
 
 #if CONFIG_EXT_REFS
-
-#define SINGLE_REFS 6
-#define COMP_REFS 5
-
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
-#define FWD_REFS 2
+#define FWD_REFS 4
 #define BWD_REFS 2
 #define SINGLE_REFS (FWD_REFS + BWD_REFS)
-#define COMP_REFS (FWD_REFS * BWD_REFS)
-
-#else  // CONFIG_BIDIR_PRED
-
+// NOTE(zoeliu): Following parameter is currently not being used
+// #define COMP_REFS (FWD_REFS * BWD_REFS)
+#else
 #define SINGLE_REFS 3
 #define COMP_REFS 2
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
 
 #if CONFIG_SUPERTX
diff --git a/vp10/common/loopfilter.h b/vp10/common/loopfilter.h
index 8fb5ef3..fc57d09 100644
--- a/vp10/common/loopfilter.h
+++ b/vp10/common/loopfilter.h
@@ -44,8 +44,8 @@
   uint8_t mode_ref_delta_enabled;
   uint8_t mode_ref_delta_update;
 
-  // 0 = Intra, Last, Last2+Last3+LAST4(CONFIG_EXT_REFS),
-  // GF, BRF(CONFIG_BIDIR_PRED), ARF
+  // 0 = Intra, Last, Last2+Last3(CONFIG_EXT_REFS),
+  // GF, BRF(CONFIG_EXT_REFS), ARF
   signed char ref_deltas[MAX_REF_FRAMES];
   signed char last_ref_deltas[MAX_REF_FRAMES];
 
diff --git a/vp10/common/onyxc_int.h b/vp10/common/onyxc_int.h
index 3ea4f3a..f7cc1f9 100644
--- a/vp10/common/onyxc_int.h
+++ b/vp10/common/onyxc_int.h
@@ -178,6 +178,7 @@
 #if CONFIG_EXT_REFS
   // frame type of the frame before last frame
   FRAME_TYPE last2_frame_type;
+  // TODO(zoeliu): To check whether last3_frame_type is still needed.
   // frame type of the frame two frames before last frame
   FRAME_TYPE last3_frame_type;
 #endif  // CONFIG_EXT_REFS
@@ -186,10 +187,10 @@
   int show_frame;
   int last_show_frame;
   int show_existing_frame;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // Flag for a frame used as a reference - not written to the bitstream
   int is_reference_frame;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Flag signaling that the frame is encoded using only INTRA modes.
   uint8_t intra_only;
@@ -278,13 +279,13 @@
   int frame_parallel_decode;  // frame-based threading.
 
   // Context probabilities for reference frame prediction
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   MV_REFERENCE_FRAME comp_fwd_ref[FWD_REFS];
   MV_REFERENCE_FRAME comp_bwd_ref[BWD_REFS];
 #else
   MV_REFERENCE_FRAME comp_fixed_ref;
   MV_REFERENCE_FRAME comp_var_ref[COMP_REFS];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   REFERENCE_MODE reference_mode;
 
   FRAME_CONTEXT *fc;  /* this frame entropy */
diff --git a/vp10/common/pred_common.c b/vp10/common/pred_common.c
index 37ae288..dd9be87 100644
--- a/vp10/common/pred_common.c
+++ b/vp10/common/pred_common.c
@@ -177,10 +177,10 @@
   }
 }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 
-#define CHECK_COMP_BWD_REF(ref_frame) \
-  (((ref_frame) == cm->comp_bwd_ref[0]) || ((ref_frame) == cm->comp_bwd_ref[1]))
+#define CHECK_BWDREF_OR_ALTREF(ref_frame) \
+  (((ref_frame) == BWDREF_FRAME) || ((ref_frame) == ALTREF_FRAME))
 
 int vp10_get_reference_mode_context(const VP10_COMMON *cm,
                                     const MACROBLOCKD *xd) {
@@ -189,6 +189,9 @@
   const MB_MODE_INFO *const left_mbmi = xd->left_mbmi;
   const int has_above = xd->up_available;
   const int has_left = xd->left_available;
+
+  (void)cm;
+
   // Note:
   // The mode info data structure has a one element border above and to the
   // left of the entries corresponding to real macroblocks.
@@ -196,15 +199,15 @@
   if (has_above && has_left) {  // both edges available
     if (!has_second_ref(above_mbmi) && !has_second_ref(left_mbmi))
       // neither edge uses comp pred (0/1)
-      ctx = CHECK_COMP_BWD_REF(above_mbmi->ref_frame[0]) ^
-            CHECK_COMP_BWD_REF(left_mbmi->ref_frame[0]);
+      ctx = CHECK_BWDREF_OR_ALTREF(above_mbmi->ref_frame[0]) ^
+            CHECK_BWDREF_OR_ALTREF(left_mbmi->ref_frame[0]);
     else if (!has_second_ref(above_mbmi))
       // one of two edges uses comp pred (2/3)
-      ctx = 2 + (CHECK_COMP_BWD_REF(above_mbmi->ref_frame[0]) ||
+      ctx = 2 + (CHECK_BWDREF_OR_ALTREF(above_mbmi->ref_frame[0]) ||
                  !is_inter_block(above_mbmi));
     else if (!has_second_ref(left_mbmi))
       // one of two edges uses comp pred (2/3)
-      ctx = 2 + (CHECK_COMP_BWD_REF(left_mbmi->ref_frame[0]) ||
+      ctx = 2 + (CHECK_BWDREF_OR_ALTREF(left_mbmi->ref_frame[0]) ||
                  !is_inter_block(left_mbmi));
     else  // both edges use comp pred (4)
       ctx = 4;
@@ -213,7 +216,7 @@
 
     if (!has_second_ref(edge_mbmi))
       // edge does not use comp pred (0/1)
-      ctx = CHECK_COMP_BWD_REF(edge_mbmi->ref_frame[0]);
+      ctx = CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]);
     else
       // edge uses comp pred (3)
       ctx = 3;
@@ -224,7 +227,7 @@
   return ctx;
 }
 
-#else  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#else  // CONFIG_EXT_REFS
 
 int vp10_get_reference_mode_context(const VP10_COMMON *cm,
                                     const MACROBLOCKD *xd) {
@@ -268,26 +271,25 @@
   return ctx;
 }
 
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 #if CONFIG_EXT_REFS
 
 // TODO(zoeliu): Future work will be conducted to optimize the context design
-// for the coding of the reference frames.
+//               for the coding of the reference frames.
 
 #define CHECK_LAST_OR_LAST2(ref_frame) \
   ((ref_frame == LAST_FRAME) || (ref_frame == LAST2_FRAME))
 
-#define CHECK_GOLDEN_LAST3_LAST4(ref_frame) \
-  ((ref_frame == GOLDEN_FRAME) || (ref_frame == LAST3_FRAME) || \
-  (ref_frame == LAST4_FRAME))
+#define CHECK_GOLDEN_OR_LAST3(ref_frame) \
+  ((ref_frame == GOLDEN_FRAME) || (ref_frame == LAST3_FRAME))
 
 // Returns a context number for the given MB prediction signal
-// Signal the first reference frame for a compound mode is either
-// GOLDEN/LAST3/LAST4, or LAST/LAST2.
+// Signal the first reference frame for a compound mode be either
+// GOLDEN/LAST3, or LAST/LAST2.
 //
 // NOTE(zoeliu): The probability of ref_frame[0] is either
-//               GOLDEN_FRAME/LAST3_FRAME/LAST4_FRAME.
+//               GOLDEN_FRAME or LAST3_FRAME.
 int vp10_get_pred_context_comp_ref_p(const VP10_COMMON *cm,
                                      const MACROBLOCKD *xd) {
   int pred_context;
@@ -300,8 +302,8 @@
   // The mode info data structure has a one element border above and to the
   // left of the entries correpsonding to real macroblocks.
   // The prediction flags in these dummy entries are initialised to 0.
-  const int fix_ref_idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref];
-  const int var_ref_idx = !fix_ref_idx;
+  const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]];
+  const int fwd_ref_sign_idx = !bwd_ref_sign_idx;
 
   if (above_in_image && left_in_image) {  // both edges available
     const int above_intra = !is_inter_block(above_mbmi);
@@ -314,51 +316,46 @@
 
       if (!has_second_ref(edge_mbmi))  // single pred (1/3)
         pred_context = 1 +
-            2 * (!CHECK_GOLDEN_LAST3_LAST4(edge_mbmi->ref_frame[0]));
+            2 * (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]));
       else  // comp pred (1/3)
-        pred_context = 1 +
-            2 * (!CHECK_GOLDEN_LAST3_LAST4(edge_mbmi->ref_frame[var_ref_idx]));
+        pred_context = 1 + 2 *
+            (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[fwd_ref_sign_idx]));
     } else {  // inter/inter
       const int l_sg = !has_second_ref(left_mbmi);
       const int a_sg = !has_second_ref(above_mbmi);
-      const MV_REFERENCE_FRAME vrfa = a_sg ? above_mbmi->ref_frame[0]
-                                           : above_mbmi->ref_frame[var_ref_idx];
-      const MV_REFERENCE_FRAME vrfl = l_sg ? left_mbmi->ref_frame[0]
-                                           : left_mbmi->ref_frame[var_ref_idx];
+      const MV_REFERENCE_FRAME frfa = a_sg ?
+          above_mbmi->ref_frame[0] : above_mbmi->ref_frame[fwd_ref_sign_idx];
+      const MV_REFERENCE_FRAME frfl = l_sg ?
+          left_mbmi->ref_frame[0] : left_mbmi->ref_frame[fwd_ref_sign_idx];
 
-      if (vrfa == vrfl && CHECK_GOLDEN_LAST3_LAST4(vrfa)) {
+      if (frfa == frfl && CHECK_GOLDEN_OR_LAST3(frfa)) {
         pred_context = 0;
       } else if (l_sg && a_sg) {  // single/single
-        if ((vrfa == ALTREF_FRAME && CHECK_LAST_OR_LAST2(vrfl)) ||
-            (vrfl == ALTREF_FRAME && CHECK_LAST_OR_LAST2(vrfa))) {
+        if ((CHECK_BWDREF_OR_ALTREF(frfa) && CHECK_LAST_OR_LAST2(frfl)) ||
+            (CHECK_BWDREF_OR_ALTREF(frfl) && CHECK_LAST_OR_LAST2(frfa))) {
           pred_context = 4;
-        } else if (vrfa == vrfl || (CHECK_LAST_OR_LAST2(vrfa) &&
-                                    CHECK_LAST_OR_LAST2(vrfl))) {
-          pred_context = 3;
-        } else {  // Either vrfa or vrfl is GOLDEN / LAST3 / LAST4
-          // NOTE(zoeliu): Following assert may be removed once confirmed.
-          assert(CHECK_GOLDEN_LAST3_LAST4(vrfa) ||
-                 CHECK_GOLDEN_LAST3_LAST4(vrfl));
+        } else if (CHECK_GOLDEN_OR_LAST3(frfa) ||
+                   CHECK_GOLDEN_OR_LAST3(frfl)) {
           pred_context = 1;
+        } else {
+          pred_context = 3;
         }
       } else if (l_sg || a_sg) {  // single/comp
-        const MV_REFERENCE_FRAME vrfc = l_sg ? vrfa : vrfl;
-        const MV_REFERENCE_FRAME rfs = a_sg ? vrfa : vrfl;
+        const MV_REFERENCE_FRAME frfc = l_sg ? frfa : frfl;
+        const MV_REFERENCE_FRAME rfs = a_sg ? frfa : frfl;
 
-        if (CHECK_GOLDEN_LAST3_LAST4(vrfc) && !CHECK_GOLDEN_LAST3_LAST4(rfs))
+        if (CHECK_GOLDEN_OR_LAST3(frfc) && !CHECK_GOLDEN_OR_LAST3(rfs))
           pred_context = 1;
-        else if (CHECK_GOLDEN_LAST3_LAST4(rfs) &&
-                 !CHECK_GOLDEN_LAST3_LAST4(vrfc))
+        else if (CHECK_GOLDEN_OR_LAST3(rfs) && !CHECK_GOLDEN_OR_LAST3(frfc))
           pred_context = 2;
         else
           pred_context = 4;
       } else {  // comp/comp
-        if ((CHECK_LAST_OR_LAST2(vrfa) && CHECK_LAST_OR_LAST2(vrfl))) {
+        if ((CHECK_LAST_OR_LAST2(frfa) && CHECK_LAST_OR_LAST2(frfl))) {
           pred_context = 4;
         } else {
           // NOTE(zoeliu): Following assert may be removed once confirmed.
-          assert(CHECK_GOLDEN_LAST3_LAST4(vrfa) ||
-                 CHECK_GOLDEN_LAST3_LAST4(vrfl));
+          assert(CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl));
           pred_context = 2;
         }
       }
@@ -370,10 +367,10 @@
       pred_context = 2;
     } else {
       if (has_second_ref(edge_mbmi))
-        pred_context =
-            4 * (!CHECK_GOLDEN_LAST3_LAST4(edge_mbmi->ref_frame[var_ref_idx]));
+        pred_context = 4 *
+            (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[fwd_ref_sign_idx]));
       else
-        pred_context = 3 * (!CHECK_GOLDEN_LAST3_LAST4(edge_mbmi->ref_frame[0]));
+        pred_context = 3 * (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]));
     }
   } else {  // no edges available (2)
     pred_context = 2;
@@ -385,7 +382,7 @@
 }
 
 // Returns a context number for the given MB prediction signal
-// Signal the first reference frame for a compound mode is LAST,
+// Signal the first reference frame for a compound mode be LAST,
 // conditioning on that it is known either LAST/LAST2.
 //
 // NOTE(zoeliu): The probability of ref_frame[0] is LAST_FRAME,
@@ -402,304 +399,6 @@
   // The mode info data structure has a one element border above and to the
   // left of the entries correpsonding to real macroblocks.
   // The prediction flags in these dummy entries are initialised to 0.
-  const int fix_ref_idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref];
-  const int var_ref_idx = !fix_ref_idx;
-
-  if (above_in_image && left_in_image) {  // both edges available
-    const int above_intra = !is_inter_block(above_mbmi);
-    const int left_intra = !is_inter_block(left_mbmi);
-
-    if (above_intra && left_intra) {  // intra/intra (2)
-      pred_context = 2;
-    } else if (above_intra || left_intra) {  // intra/inter
-      const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
-
-      if (!has_second_ref(edge_mbmi))  // single pred (1/3)
-        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != LAST_FRAME);
-      else  // comp pred (1/3)
-        pred_context = 1 + 2 * (edge_mbmi->ref_frame[var_ref_idx]
-                                != LAST_FRAME);
-    } else {  // inter/inter
-      const int l_sg = !has_second_ref(left_mbmi);
-      const int a_sg = !has_second_ref(above_mbmi);
-      const MV_REFERENCE_FRAME vrfa = a_sg ? above_mbmi->ref_frame[0]
-                                           : above_mbmi->ref_frame[var_ref_idx];
-      const MV_REFERENCE_FRAME vrfl = l_sg ? left_mbmi->ref_frame[0]
-                                           : left_mbmi->ref_frame[var_ref_idx];
-
-      if (vrfa == vrfl && vrfa == LAST_FRAME)
-        pred_context = 0;
-      else if (l_sg && a_sg) {  // single/single
-        if (vrfa == LAST_FRAME || vrfl == LAST_FRAME)
-          pred_context = 1;
-        else if (CHECK_GOLDEN_LAST3_LAST4(vrfa) ||
-                 CHECK_GOLDEN_LAST3_LAST4(vrfl))
-          pred_context = 2 + (vrfa != vrfl);
-        else if (vrfa == vrfl)
-          pred_context = 3;
-        else
-          pred_context = 4;
-      } else if (l_sg || a_sg) {  // single/comp
-        const MV_REFERENCE_FRAME vrfc = l_sg ? vrfa : vrfl;
-        const MV_REFERENCE_FRAME rfs = a_sg ? vrfa : vrfl;
-
-        if (vrfc == LAST_FRAME && rfs != LAST_FRAME)
-          pred_context = 1;
-        else if (rfs == LAST_FRAME && vrfc != LAST_FRAME)
-          pred_context = 2;
-        else
-          pred_context = 3 +
-              (vrfc == LAST2_FRAME || CHECK_GOLDEN_LAST3_LAST4(rfs));
-      } else {  // comp/comp
-        if (vrfa == LAST_FRAME || vrfl == LAST_FRAME)
-          pred_context = 2;
-        else
-          pred_context = 3 + (CHECK_GOLDEN_LAST3_LAST4(vrfa) ||
-                              CHECK_GOLDEN_LAST3_LAST4(vrfl));
-      }
-    }
-  } else if (above_in_image || left_in_image) {  // one edge available
-    const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi;
-
-    if (!is_inter_block(edge_mbmi)) {
-      pred_context = 2;
-    } else {
-      if (has_second_ref(edge_mbmi)) {
-        pred_context = 4 * (edge_mbmi->ref_frame[var_ref_idx] != LAST_FRAME);
-      } else {
-        if (edge_mbmi->ref_frame[0] == LAST_FRAME)
-          pred_context = 0;
-        else
-          pred_context = 2 + CHECK_GOLDEN_LAST3_LAST4(edge_mbmi->ref_frame[0]);
-      }
-    }
-  } else {  // no edges available (2)
-    pred_context = 2;
-  }
-
-  assert(pred_context >= 0 && pred_context < REF_CONTEXTS);
-
-  return pred_context;
-}
-
-#define CHECK_LAST3_OR_LAST4(ref_frame) \
-  ((ref_frame == LAST3_FRAME) || (ref_frame == LAST4_FRAME))
-
-// Returns a context number for the given MB prediction signal
-// Signal the first reference frame for a compound mode is GOLDEN,
-// conditioning on that it is known either GOLDEN/LAST3/LAST4.
-//
-// NOTE(zoeliu): The probability of ref_frame[0] is GOLDEN_FRAME,
-// conditioning on it is either GOLDEN / LAST3 / LAST4.
-int vp10_get_pred_context_comp_ref_p2(const VP10_COMMON *cm,
-                                      const MACROBLOCKD *xd) {
-  int pred_context;
-  const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
-  const MB_MODE_INFO *const left_mbmi = xd->left_mbmi;
-  const int above_in_image = xd->up_available;
-  const int left_in_image = xd->left_available;
-
-  // Note:
-  // The mode info data structure has a one element border above and to the
-  // left of the entries correpsonding to real macroblocks.
-  // The prediction flags in these dummy entries are initialised to 0.
-  const int fix_ref_idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref];
-  const int var_ref_idx = !fix_ref_idx;
-
-  if (above_in_image && left_in_image) {  // both edges available
-    const int above_intra = !is_inter_block(above_mbmi);
-    const int left_intra = !is_inter_block(left_mbmi);
-
-    if (above_intra && left_intra) {  // intra/intra (2)
-      pred_context = 2;
-    } else if (above_intra || left_intra) {  // intra/inter
-      const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
-
-      if (!has_second_ref(edge_mbmi))  // single pred (1/3)
-        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != GOLDEN_FRAME);
-      else  // comp pred (1/3)
-        pred_context = 1 +
-            2 * (edge_mbmi->ref_frame[var_ref_idx] != GOLDEN_FRAME);
-    } else {  // inter/inter
-      const int l_sg = !has_second_ref(left_mbmi);
-      const int a_sg = !has_second_ref(above_mbmi);
-      const MV_REFERENCE_FRAME vrfa = a_sg ? above_mbmi->ref_frame[0]
-                                           : above_mbmi->ref_frame[var_ref_idx];
-      const MV_REFERENCE_FRAME vrfl = l_sg ? left_mbmi->ref_frame[0]
-                                           : left_mbmi->ref_frame[var_ref_idx];
-
-      if (vrfa == vrfl && vrfa == GOLDEN_FRAME)
-        pred_context = 0;
-      else if (l_sg && a_sg) {  // single/single
-        if (vrfa == GOLDEN_FRAME || vrfl == GOLDEN_FRAME)
-          pred_context = 1;
-        else if (CHECK_LAST_OR_LAST2(vrfa) || CHECK_LAST_OR_LAST2(vrfl))
-          pred_context = 2 + (vrfa != vrfl);
-        else if (vrfa == vrfl)
-          pred_context = 3;
-        else
-          pred_context = 4;
-      } else if (l_sg || a_sg) {  // single/comp
-        const MV_REFERENCE_FRAME vrfc = l_sg ? vrfa : vrfl;
-        const MV_REFERENCE_FRAME rfs = a_sg ? vrfa : vrfl;
-
-        if (vrfc == GOLDEN_FRAME && rfs != GOLDEN_FRAME)
-          pred_context = 1;
-        else if (rfs == GOLDEN_FRAME && vrfc != GOLDEN_FRAME)
-          pred_context = 2;
-        else
-          pred_context = 3 +
-              (CHECK_LAST3_OR_LAST4(vrfc) || CHECK_LAST_OR_LAST2(rfs));
-      } else {  // comp/comp
-        if (vrfa == GOLDEN_FRAME || vrfl == GOLDEN_FRAME)
-          pred_context = 2;
-        else
-          pred_context = 3 +
-              (CHECK_LAST_OR_LAST2(vrfa) || CHECK_LAST_OR_LAST2(vrfl));
-      }
-    }
-  } else if (above_in_image || left_in_image) {  // one edge available
-    const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi;
-
-    if (!is_inter_block(edge_mbmi)) {
-      pred_context = 2;
-    } else {
-      if (has_second_ref(edge_mbmi)) {
-        pred_context = 4 * (edge_mbmi->ref_frame[var_ref_idx] != GOLDEN_FRAME);
-      } else {
-        if (edge_mbmi->ref_frame[0] == GOLDEN_FRAME)
-          pred_context = 0;
-        else
-          pred_context = 2 + CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]);
-      }
-    }
-  } else {  // no edges available (2)
-    pred_context = 2;
-  }
-
-  assert(pred_context >= 0 && pred_context < REF_CONTEXTS);
-
-  return pred_context;
-}
-
-#define CHECK_LAST_LAST2_GOLDEN(ref_frame) \
-  ((ref_frame == LAST_FRAME) || (ref_frame == LAST2_FRAME) || \
-  (ref_frame == GOLDEN_FRAME))
-
-// Returns a context number for the given MB prediction signal
-// Signal the first reference frame for a compound mode is LAST3,
-// conditioning on that it is known either LAST3/LAST4.
-//
-// NOTE(zoeliu): The probability of ref_frame[0] is LAST3_FRAME,
-// conditioning on it is either LAST3 / LAST4.
-int vp10_get_pred_context_comp_ref_p3(const VP10_COMMON *cm,
-                                      const MACROBLOCKD *xd) {
-  int pred_context;
-  const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
-  const MB_MODE_INFO *const left_mbmi = xd->left_mbmi;
-  const int above_in_image = xd->up_available;
-  const int left_in_image = xd->left_available;
-
-  // Note:
-  // The mode info data structure has a one element border above and to the
-  // left of the entries correpsonding to real macroblocks.
-  // The prediction flags in these dummy entries are initialised to 0.
-  const int fix_ref_idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref];
-  const int var_ref_idx = !fix_ref_idx;
-
-  if (above_in_image && left_in_image) {  // both edges available
-    const int above_intra = !is_inter_block(above_mbmi);
-    const int left_intra = !is_inter_block(left_mbmi);
-
-    if (above_intra && left_intra) {  // intra/intra (2)
-      pred_context = 2;
-    } else if (above_intra || left_intra) {  // intra/inter
-      const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
-
-      if (!has_second_ref(edge_mbmi))  // single pred (1/3)
-        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != LAST3_FRAME);
-      else  // comp pred (1/3)
-        pred_context = 1 +
-            2 * (edge_mbmi->ref_frame[var_ref_idx] != LAST3_FRAME);
-    } else {  // inter/inter
-      const int l_sg = !has_second_ref(left_mbmi);
-      const int a_sg = !has_second_ref(above_mbmi);
-      const MV_REFERENCE_FRAME vrfa = a_sg ? above_mbmi->ref_frame[0]
-                                           : above_mbmi->ref_frame[var_ref_idx];
-      const MV_REFERENCE_FRAME vrfl = l_sg ? left_mbmi->ref_frame[0]
-                                           : left_mbmi->ref_frame[var_ref_idx];
-
-      if (vrfa == vrfl && vrfa == LAST3_FRAME)
-        pred_context = 0;
-      else if (l_sg && a_sg) {  // single/single
-        if (vrfa == LAST3_FRAME || vrfl == LAST3_FRAME)
-          pred_context = 1;
-        else if (CHECK_LAST_LAST2_GOLDEN(vrfa) || CHECK_LAST_LAST2_GOLDEN(vrfl))
-          pred_context = 2 + (vrfa != vrfl);
-        else if (vrfa == vrfl)
-          pred_context = 3;
-        else
-          pred_context = 4;
-      } else if (l_sg || a_sg) {  // single/comp
-        const MV_REFERENCE_FRAME vrfc = l_sg ? vrfa : vrfl;
-        const MV_REFERENCE_FRAME rfs = a_sg ? vrfa : vrfl;
-
-        if (vrfc == LAST3_FRAME && rfs != LAST3_FRAME)
-          pred_context = 1;
-        else if (rfs == LAST3_FRAME && vrfc != LAST3_FRAME)
-          pred_context = 2;
-        else
-          pred_context = 3 +
-              (vrfc == LAST4_FRAME || CHECK_LAST_LAST2_GOLDEN(rfs));
-      } else {  // comp/comp
-        if (vrfa == LAST3_FRAME || vrfl == LAST3_FRAME)
-          pred_context = 2;
-        else
-          pred_context = 3 +
-              (CHECK_LAST_LAST2_GOLDEN(vrfa) || CHECK_LAST_LAST2_GOLDEN(vrfl));
-      }
-    }
-  } else if (above_in_image || left_in_image) {  // one edge available
-    const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi;
-
-    if (!is_inter_block(edge_mbmi)) {
-      pred_context = 2;
-    } else {
-      if (has_second_ref(edge_mbmi)) {
-        pred_context = 4 * (edge_mbmi->ref_frame[var_ref_idx] != LAST3_FRAME);
-      } else {
-        if (edge_mbmi->ref_frame[0] == LAST3_FRAME)
-          pred_context = 0;
-        else
-          pred_context = 2 + CHECK_LAST_LAST2_GOLDEN(edge_mbmi->ref_frame[0]);
-      }
-    }
-  } else {  // no edges available (2)
-    pred_context = 2;
-  }
-
-  assert(pred_context >= 0 && pred_context < REF_CONTEXTS);
-
-  return pred_context;
-}
-
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
-
-// Returns a context number for the given MB prediction signal
-int vp10_get_pred_context_comp_ref_p(const VP10_COMMON *cm,
-                                     const MACROBLOCKD *xd) {
-  int pred_context;
-  const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
-  const MB_MODE_INFO *const left_mbmi = xd->left_mbmi;
-  const int above_in_image = xd->up_available;
-  const int left_in_image = xd->left_available;
-
-  // Note:
-  // The mode info data structure has a one element border above and to the
-  // left of the entries corresponding to real macroblocks.
-  // The prediction flags in these dummy entries are initialized to 0.
   const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]];
   const int fwd_ref_sign_idx = !bwd_ref_sign_idx;
 
@@ -713,10 +412,10 @@
       const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
 
       if (!has_second_ref(edge_mbmi))  // single pred (1/3)
-        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != cm->comp_fwd_ref[1]);
+        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != LAST_FRAME);
       else  // comp pred (1/3)
         pred_context = 1 + 2 * (edge_mbmi->ref_frame[fwd_ref_sign_idx]
-                                != cm->comp_fwd_ref[1]);
+                                != LAST_FRAME);
     } else {  // inter/inter
       const int l_sg = !has_second_ref(left_mbmi);
       const int a_sg = !has_second_ref(above_mbmi);
@@ -725,29 +424,35 @@
       const MV_REFERENCE_FRAME frfl = l_sg ?
           left_mbmi->ref_frame[0] : left_mbmi->ref_frame[fwd_ref_sign_idx];
 
-      if (frfa == frfl && frfa == cm->comp_fwd_ref[1]) {
+      if (frfa == frfl && frfa == LAST_FRAME)
         pred_context = 0;
-      } else if (l_sg && a_sg) {  // single/single
-        if ((frfa != frfl) &&
-            (frfa != cm->comp_fwd_ref[1]) && (frfl != cm->comp_fwd_ref[1]))
-          pred_context = 4;
-        else if (frfa == frfl)
+      else if (l_sg && a_sg) {  // single/single
+        if (frfa == LAST_FRAME || frfl == LAST_FRAME)
+          pred_context = 1;
+        else if (CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl))
+          pred_context = 2 + (frfa != frfl);
+        else if (frfa == frfl ||
+                 (CHECK_BWDREF_OR_ALTREF(frfa) && CHECK_BWDREF_OR_ALTREF(frfl)))
           pred_context = 3;
         else
-          pred_context = 1;
+          pred_context = 4;
       } else if (l_sg || a_sg) {  // single/comp
         const MV_REFERENCE_FRAME frfc = l_sg ? frfa : frfl;
         const MV_REFERENCE_FRAME rfs = a_sg ? frfa : frfl;
-        if (frfc == cm->comp_fwd_ref[1] && rfs != cm->comp_fwd_ref[1])
+
+        if (frfc == LAST_FRAME && rfs != LAST_FRAME)
           pred_context = 1;
-        else if (rfs == cm->comp_fwd_ref[1] && frfc != cm->comp_fwd_ref[1])
+        else if (rfs == LAST_FRAME && frfc != LAST_FRAME)
           pred_context = 2;
         else
-          pred_context = 4;
-      } else if (frfa == frfl) {  // comp/comp
-        pred_context = 4;
-      } else {
-        pred_context = 2;
+          pred_context = 3 +
+              (frfc == LAST2_FRAME || CHECK_GOLDEN_OR_LAST3(rfs));
+      } else {  // comp/comp
+        if (frfa == LAST_FRAME || frfl == LAST_FRAME)
+          pred_context = 2;
+        else
+          pred_context = 3 +
+              (CHECK_GOLDEN_OR_LAST3(frfa) || CHECK_GOLDEN_OR_LAST3(frfl));
       }
     }
   } else if (above_in_image || left_in_image) {  // one edge available
@@ -756,15 +461,119 @@
     if (!is_inter_block(edge_mbmi)) {
       pred_context = 2;
     } else {
-      if (has_second_ref(edge_mbmi))
-        pred_context = 4 * (edge_mbmi->ref_frame[fwd_ref_sign_idx]
-                            != cm->comp_fwd_ref[1]);
-      else
-        pred_context = 3 * (edge_mbmi->ref_frame[0] != cm->comp_fwd_ref[1]);
+      if (has_second_ref(edge_mbmi)) {
+        pred_context = 4 *
+            (edge_mbmi->ref_frame[fwd_ref_sign_idx] != LAST_FRAME);
+      } else {
+        if (edge_mbmi->ref_frame[0] == LAST_FRAME)
+          pred_context = 0;
+        else
+          pred_context = 2 + CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]);
+      }
     }
   } else {  // no edges available (2)
     pred_context = 2;
   }
+
+  assert(pred_context >= 0 && pred_context < REF_CONTEXTS);
+
+  return pred_context;
+}
+
+// Returns a context number for the given MB prediction signal
+// Signal the first reference frame for a compound mode be GOLDEN,
+// conditioning on that it is known either GOLDEN or LAST3.
+//
+// NOTE(zoeliu): The probability of ref_frame[0] is GOLDEN_FRAME,
+// conditioning on it is either GOLDEN or LAST3.
+int vp10_get_pred_context_comp_ref_p2(const VP10_COMMON *cm,
+                                      const MACROBLOCKD *xd) {
+  int pred_context;
+  const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
+  const MB_MODE_INFO *const left_mbmi = xd->left_mbmi;
+  const int above_in_image = xd->up_available;
+  const int left_in_image = xd->left_available;
+
+  // Note:
+  // The mode info data structure has a one element border above and to the
+  // left of the entries correpsonding to real macroblocks.
+  // The prediction flags in these dummy entries are initialised to 0.
+  const int bwd_ref_sign_idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]];
+  const int fwd_ref_sign_idx = !bwd_ref_sign_idx;
+
+  if (above_in_image && left_in_image) {  // both edges available
+    const int above_intra = !is_inter_block(above_mbmi);
+    const int left_intra = !is_inter_block(left_mbmi);
+
+    if (above_intra && left_intra) {  // intra/intra (2)
+      pred_context = 2;
+    } else if (above_intra || left_intra) {  // intra/inter
+      const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
+
+      if (!has_second_ref(edge_mbmi))  // single pred (1/3)
+        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] != GOLDEN_FRAME);
+      else  // comp pred (1/3)
+        pred_context = 1 + 2 * (edge_mbmi->ref_frame[fwd_ref_sign_idx]
+                                != GOLDEN_FRAME);
+    } else {  // inter/inter
+      const int l_sg = !has_second_ref(left_mbmi);
+      const int a_sg = !has_second_ref(above_mbmi);
+      const MV_REFERENCE_FRAME frfa = a_sg ?
+          above_mbmi->ref_frame[0] : above_mbmi->ref_frame[fwd_ref_sign_idx];
+      const MV_REFERENCE_FRAME frfl = l_sg ?
+          left_mbmi->ref_frame[0] : left_mbmi->ref_frame[fwd_ref_sign_idx];
+
+      if (frfa == frfl && frfa == GOLDEN_FRAME)
+        pred_context = 0;
+      else if (l_sg && a_sg) {  // single/single
+        if (frfa == GOLDEN_FRAME || frfl == GOLDEN_FRAME)
+          pred_context = 1;
+        else if (CHECK_LAST_OR_LAST2(frfa) || CHECK_LAST_OR_LAST2(frfl))
+          pred_context = 2 + (frfa != frfl);
+        else if (frfa == frfl ||
+                 (CHECK_BWDREF_OR_ALTREF(frfa) && CHECK_BWDREF_OR_ALTREF(frfl)))
+          pred_context = 3;
+        else
+          pred_context = 4;
+      } else if (l_sg || a_sg) {  // single/comp
+        const MV_REFERENCE_FRAME frfc = l_sg ? frfa : frfl;
+        const MV_REFERENCE_FRAME rfs = a_sg ? frfa : frfl;
+
+        if (frfc == GOLDEN_FRAME && rfs != GOLDEN_FRAME)
+          pred_context = 1;
+        else if (rfs == GOLDEN_FRAME && frfc != GOLDEN_FRAME)
+          pred_context = 2;
+        else
+          pred_context = 3 +
+              (frfc == LAST3_FRAME || CHECK_LAST_OR_LAST2(rfs));
+      } else {  // comp/comp
+        if (frfa == GOLDEN_FRAME || frfl == GOLDEN_FRAME)
+          pred_context = 2;
+        else
+          pred_context = 3 +
+              (CHECK_LAST_OR_LAST2(frfa) || CHECK_LAST_OR_LAST2(frfl));
+      }
+    }
+  } else if (above_in_image || left_in_image) {  // one edge available
+    const MB_MODE_INFO *edge_mbmi = above_in_image ? above_mbmi : left_mbmi;
+
+    if (!is_inter_block(edge_mbmi)) {
+      pred_context = 2;
+    } else {
+      if (has_second_ref(edge_mbmi)) {
+        pred_context = 4 *
+            (edge_mbmi->ref_frame[fwd_ref_sign_idx] != GOLDEN_FRAME);
+      } else {
+        if (edge_mbmi->ref_frame[0] == GOLDEN_FRAME)
+          pred_context = 0;
+        else
+          pred_context = 2 + CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]);
+      }
+    }
+  } else {  // no edges available (2)
+    pred_context = 2;
+  }
+
   assert(pred_context >= 0 && pred_context < REF_CONTEXTS);
 
   return pred_context;
@@ -814,7 +623,7 @@
       const MV_REFERENCE_FRAME a_frf = !a_comp ?
           above_mbmi->ref_frame[0] : above_mbmi->ref_frame[fwd_ref_sign_idx];
 
-      if (l_comp && a_comp) {
+      if (l_comp && a_comp) {  // comp/comp
         if (l_brf == a_brf && l_brf == cm->comp_bwd_ref[1]) {
           pred_context = 0;
         } else if (l_brf == cm->comp_bwd_ref[1] ||
@@ -825,7 +634,7 @@
           assert(l_brf == a_brf && l_brf != cm->comp_bwd_ref[1]);
           pred_context = 3;
         }
-      } else if (!l_comp && !a_comp) {
+      } else if (!l_comp && !a_comp) {  // single/single
         if (l_frf == a_frf && l_frf == cm->comp_bwd_ref[1]) {
           pred_context = 0;
         } else if (l_frf == cm->comp_bwd_ref[1] ||
@@ -838,7 +647,7 @@
                  l_frf != cm->comp_bwd_ref[1] && a_frf != cm->comp_bwd_ref[1]);
           pred_context = 4;
         }
-      } else {
+      } else {  // comp/single
         assert((l_comp && !a_comp) || (!l_comp && a_comp));
 
         if ((l_comp && l_brf == cm->comp_bwd_ref[1] &&
@@ -877,7 +686,7 @@
   return pred_context;
 }
 
-#else  // CONFIG_BIDIR_PRED
+#else  // CONFIG_EXT_REFS
 
 // Returns a context number for the given MB prediction signal
 int vp10_get_pred_context_comp_ref_p(const VP10_COMMON *cm,
@@ -962,19 +771,14 @@
   return pred_context;
 }
 
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
 
 #if CONFIG_EXT_REFS
 
-#define CHECK_GOLDEN_OR_ALTREF(ref_frame) \
-  ((ref_frame == GOLDEN_FRAME) || (ref_frame == ALTREF_FRAME))
-
 // For the bit to signal whether the single reference is a ALTREF_FRAME
-// or a GOLDEN_FRAME.
+// or a BWDREF_FRAME.
 //
-// NOTE(zoeliu): The probability of ref_frame[0] is ALTREF/GOLDEN.
+// NOTE(zoeliu): The probability of ref_frame[0] is ALTREF/BWDREF.
 int vp10_get_pred_context_single_ref_p1(const MACROBLOCKD *xd) {
   int pred_context;
   const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
@@ -996,10 +800,10 @@
       const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
 
       if (!has_second_ref(edge_mbmi))
-        pred_context = 4 * (!CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]));
+        pred_context = 4 * (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]));
       else
-        pred_context = 1 + (!CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]) ||
-                            !CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[1]));
+        pred_context = 1 + (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]) ||
+                            !CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[1]));
     } else {  // inter/inter
       const int above_has_second = has_second_ref(above_mbmi);
       const int left_has_second  = has_second_ref(left_mbmi);
@@ -1010,24 +814,24 @@
       const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1];
 
       if (above_has_second && left_has_second) {
-        pred_context = 1 + (!CHECK_GOLDEN_OR_ALTREF(above0) ||
-                            !CHECK_GOLDEN_OR_ALTREF(above1) ||
-                            !CHECK_GOLDEN_OR_ALTREF(left0) ||
-                            !CHECK_GOLDEN_OR_ALTREF(left1));
+        pred_context = 1 + (!CHECK_BWDREF_OR_ALTREF(above0) ||
+                            !CHECK_BWDREF_OR_ALTREF(above1) ||
+                            !CHECK_BWDREF_OR_ALTREF(left0) ||
+                            !CHECK_BWDREF_OR_ALTREF(left1));
       } else if (above_has_second || left_has_second) {
         const MV_REFERENCE_FRAME rfs = !above_has_second ? above0 : left0;
         const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0;
         const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1;
 
-        if (!CHECK_GOLDEN_OR_ALTREF(rfs))
-          pred_context = 3 + (!CHECK_GOLDEN_OR_ALTREF(crf1) ||
-                              !CHECK_GOLDEN_OR_ALTREF(crf2));
+        if (!CHECK_BWDREF_OR_ALTREF(rfs))
+          pred_context = 3 + (!CHECK_BWDREF_OR_ALTREF(crf1) ||
+                              !CHECK_BWDREF_OR_ALTREF(crf2));
         else
-          pred_context = !CHECK_GOLDEN_OR_ALTREF(crf1) ||
-                         !CHECK_GOLDEN_OR_ALTREF(crf2);
+          pred_context = !CHECK_BWDREF_OR_ALTREF(crf1) ||
+                         !CHECK_BWDREF_OR_ALTREF(crf2);
       } else {
-        pred_context = 2 * (!CHECK_GOLDEN_OR_ALTREF(above0)) +
-                       2 * (!CHECK_GOLDEN_OR_ALTREF(left0));
+        pred_context = 2 * (!CHECK_BWDREF_OR_ALTREF(above0)) +
+                       2 * (!CHECK_BWDREF_OR_ALTREF(left0));
       }
     }
   } else if (has_above || has_left) {  // one edge available
@@ -1036,10 +840,10 @@
       pred_context = 2;
     } else {  // inter
       if (!has_second_ref(edge_mbmi))
-        pred_context = 4 * (!CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]));
+        pred_context = 4 * (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]));
       else
-        pred_context = 1 + (!CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]) ||
-                            !CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[1]));
+        pred_context = 1 + (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]) ||
+                            !CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[1]));
     }
   } else {  // no edges available
     pred_context = 2;
@@ -1050,10 +854,10 @@
 }
 
 // For the bit to signal whether the single reference is ALTREF_FRAME or
-// GOLDEN_FRAME, knowing that it shall be either of these 2 choices.
+// BWDREF_FRAME, knowing that it shall be either of these 2 choices.
 //
 // NOTE(zoeliu): The probability of ref_frame[0] is ALTREF_FRAME, conditioning
-// on it is either ALTREF_FRAME/GOLDEN_FRAME.
+// on it is either ALTREF_FRAME/BWDREF_FRAME.
 int vp10_get_pred_context_single_ref_p2(const MACROBLOCKD *xd) {
   int pred_context;
   const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
@@ -1074,13 +878,13 @@
     } else if (above_intra || left_intra) {  // intra/inter or inter/intra
       const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
       if (!has_second_ref(edge_mbmi)) {
-        if (!CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]))
+        if (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]))
           pred_context = 3;
         else
-          pred_context = 4 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME);
+          pred_context = 4 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME);
       } else {
-        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME ||
-                                edge_mbmi->ref_frame[1] == GOLDEN_FRAME);
+        pred_context = 1 + 2 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME ||
+                                edge_mbmi->ref_frame[1] == BWDREF_FRAME);
       }
     } else {  // inter/inter
       const int above_has_second = has_second_ref(above_mbmi);
@@ -1092,10 +896,10 @@
 
       if (above_has_second && left_has_second) {
         if (above0 == left0 && above1 == left1)
-          pred_context = 3 * (above0 == GOLDEN_FRAME ||
-                              above1 == GOLDEN_FRAME ||
-                              left0 == GOLDEN_FRAME ||
-                              left1 == GOLDEN_FRAME);
+          pred_context = 3 * (above0 == BWDREF_FRAME ||
+                              above1 == BWDREF_FRAME ||
+                              left0 == BWDREF_FRAME ||
+                              left1 == BWDREF_FRAME);
         else
           pred_context = 2;
       } else if (above_has_second || left_has_second) {
@@ -1103,23 +907,23 @@
         const MV_REFERENCE_FRAME crf1 = above_has_second ? above0 : left0;
         const MV_REFERENCE_FRAME crf2 = above_has_second ? above1 : left1;
 
-        if (rfs == GOLDEN_FRAME)
-          pred_context = 3 + (crf1 == GOLDEN_FRAME || crf2 == GOLDEN_FRAME);
+        if (rfs == BWDREF_FRAME)
+          pred_context = 3 + (crf1 == BWDREF_FRAME || crf2 == BWDREF_FRAME);
         else if (rfs == ALTREF_FRAME)
-          pred_context = (crf1 == GOLDEN_FRAME || crf2 == GOLDEN_FRAME);
+          pred_context = (crf1 == BWDREF_FRAME || crf2 == BWDREF_FRAME);
         else
-          pred_context = 1 + 2 * (crf1 == GOLDEN_FRAME || crf2 == GOLDEN_FRAME);
+          pred_context = 1 + 2 * (crf1 == BWDREF_FRAME || crf2 == BWDREF_FRAME);
       } else {
-        if (!CHECK_GOLDEN_OR_ALTREF(above0) && !CHECK_GOLDEN_OR_ALTREF(left0)) {
+        if (!CHECK_BWDREF_OR_ALTREF(above0) && !CHECK_BWDREF_OR_ALTREF(left0)) {
           pred_context = 2 + (above0 == left0);
-        } else if (!CHECK_GOLDEN_OR_ALTREF(above0) ||
-                   !CHECK_GOLDEN_OR_ALTREF(left0)) {
+        } else if (!CHECK_BWDREF_OR_ALTREF(above0) ||
+                   !CHECK_BWDREF_OR_ALTREF(left0)) {
           const MV_REFERENCE_FRAME edge0 =
-              !CHECK_GOLDEN_OR_ALTREF(above0) ? left0 : above0;
-          pred_context = 4 * (edge0 == GOLDEN_FRAME);
+              !CHECK_BWDREF_OR_ALTREF(above0) ? left0 : above0;
+          pred_context = 4 * (edge0 == BWDREF_FRAME);
         } else {
-          pred_context = 2 * (above0 == GOLDEN_FRAME) +
-                         2 * (left0  == GOLDEN_FRAME);
+          pred_context = 2 * (above0 == BWDREF_FRAME) +
+                         2 * (left0  == BWDREF_FRAME);
         }
       }
     }
@@ -1127,14 +931,14 @@
     const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi;
 
     if (!is_inter_block(edge_mbmi) ||
-        (!CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]) &&
+        (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]) &&
          !has_second_ref(edge_mbmi)))
       pred_context = 2;
     else if (!has_second_ref(edge_mbmi))
-      pred_context = 4 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME);
+      pred_context = 4 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME);
     else
-      pred_context = 3 * (edge_mbmi->ref_frame[0] == GOLDEN_FRAME ||
-                          edge_mbmi->ref_frame[1] == GOLDEN_FRAME);
+      pred_context = 3 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME ||
+                          edge_mbmi->ref_frame[1] == BWDREF_FRAME);
   } else {  // no edges available (2)
     pred_context = 2;
   }
@@ -1143,11 +947,11 @@
   return pred_context;
 }
 
-// For the bit to signal whether the single reference is LAST3/LAST4 or
+// For the bit to signal whether the single reference is LAST3/GOLDEN or
 // LAST2/LAST, knowing that it shall be either of these 2 choices.
 //
-// NOTE(zoeliu): The probability of ref_frame[0] is LAST3/LAST4, conditioning
-// on it is either LAST3/LAST4/LAST2/LAST.
+// NOTE(zoeliu): The probability of ref_frame[0] is LAST3/GOLDEN, conditioning
+// on it is either LAST3/GOLDEN/LAST2/LAST.
 int vp10_get_pred_context_single_ref_p3(const MACROBLOCKD *xd) {
   int pred_context;
   const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
@@ -1168,7 +972,7 @@
     } else if (above_intra || left_intra) {  // intra/inter or inter/intra
       const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
       if (!has_second_ref(edge_mbmi)) {
-        if (CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]))
+        if (CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]))
           pred_context = 3;
         else
           pred_context = 4 * CHECK_LAST_OR_LAST2(edge_mbmi->ref_frame[0]);
@@ -1201,19 +1005,19 @@
         if (CHECK_LAST_OR_LAST2(rfs))
           pred_context = 3 + (CHECK_LAST_OR_LAST2(crf1) ||
                               CHECK_LAST_OR_LAST2(crf2));
-        else if (rfs == LAST3_FRAME || rfs == LAST4_FRAME)
+        else if (CHECK_GOLDEN_OR_LAST3(rfs))
           pred_context = (CHECK_LAST_OR_LAST2(crf1) ||
                           CHECK_LAST_OR_LAST2(crf2));
         else
           pred_context = 1 + 2 * (CHECK_LAST_OR_LAST2(crf1) ||
                                   CHECK_LAST_OR_LAST2(crf2));
       } else {
-        if (CHECK_GOLDEN_OR_ALTREF(above0) && CHECK_GOLDEN_OR_ALTREF(left0)) {
+        if (CHECK_BWDREF_OR_ALTREF(above0) && CHECK_BWDREF_OR_ALTREF(left0)) {
           pred_context = 2 + (above0 == left0);
-        } else if (CHECK_GOLDEN_OR_ALTREF(above0) ||
-                   CHECK_GOLDEN_OR_ALTREF(left0)) {
+        } else if (CHECK_BWDREF_OR_ALTREF(above0) ||
+                   CHECK_BWDREF_OR_ALTREF(left0)) {
           const MV_REFERENCE_FRAME edge0 =
-              CHECK_GOLDEN_OR_ALTREF(above0) ? left0 : above0;
+              CHECK_BWDREF_OR_ALTREF(above0) ? left0 : above0;
           pred_context = 4 * CHECK_LAST_OR_LAST2(edge0);
         } else {
           pred_context = 2 * CHECK_LAST_OR_LAST2(above0) +
@@ -1225,7 +1029,7 @@
     const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi;
 
     if (!is_inter_block(edge_mbmi) ||
-        (CHECK_GOLDEN_OR_ALTREF(edge_mbmi->ref_frame[0]) &&
+        (CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]) &&
          !has_second_ref(edge_mbmi)))
       pred_context = 2;
     else if (!has_second_ref(edge_mbmi))
@@ -1334,11 +1138,11 @@
   return pred_context;
 }
 
-// For the bit to signal whether the single reference is LAST4_FRAME or
+// For the bit to signal whether the single reference is GOLDEN_FRAME or
 // LAST3_FRAME, knowing that it shall be either of these 2 choices.
 //
-// NOTE(zoeliu): The probability of ref_frame[0] is LAST4_FRAME, conditioning
-// on it is either LAST4_FRAME/LAST3_FRAME.
+// NOTE(zoeliu): The probability of ref_frame[0] is GOLDEN_FRAME, conditioning
+// on it is either GOLDEN_FRAME/LAST3_FRAME.
 int vp10_get_pred_context_single_ref_p5(const MACROBLOCKD *xd) {
   int pred_context;
   const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
@@ -1359,7 +1163,7 @@
     } else if (above_intra || left_intra) {  // intra/inter or inter/intra
       const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
       if (!has_second_ref(edge_mbmi)) {
-        if (!CHECK_LAST3_OR_LAST4(edge_mbmi->ref_frame[0]))
+        if (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]))
           pred_context = 3;
         else
           pred_context = 4 * (edge_mbmi->ref_frame[0] == LAST3_FRAME);
@@ -1389,18 +1193,18 @@
 
         if (rfs == LAST3_FRAME)
           pred_context = 3 + (crf1 == LAST3_FRAME || crf2 == LAST3_FRAME);
-        else if (rfs == LAST4_FRAME)
+        else if (rfs == GOLDEN_FRAME)
           pred_context = (crf1 == LAST3_FRAME || crf2 == LAST3_FRAME);
         else
           pred_context = 1 + 2 * (crf1 == LAST3_FRAME || crf2 == LAST3_FRAME);
       } else {
-        if (!CHECK_LAST3_OR_LAST4(above0) &&
-            !CHECK_LAST3_OR_LAST4(left0)) {
+        if (!CHECK_GOLDEN_OR_LAST3(above0) &&
+            !CHECK_GOLDEN_OR_LAST3(left0)) {
           pred_context = 2 + (above0 == left0);
-        } else if (!CHECK_LAST3_OR_LAST4(above0) ||
-                   !CHECK_LAST3_OR_LAST4(left0)) {
+        } else if (!CHECK_GOLDEN_OR_LAST3(above0) ||
+                   !CHECK_GOLDEN_OR_LAST3(left0)) {
           const MV_REFERENCE_FRAME edge0 =
-              !CHECK_LAST3_OR_LAST4(above0) ? left0 : above0;
+              !CHECK_GOLDEN_OR_LAST3(above0) ? left0 : above0;
           pred_context = 4 * (edge0 == LAST3_FRAME);
         } else {
           pred_context = 2 * (above0 == LAST3_FRAME) +
@@ -1412,7 +1216,7 @@
     const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi;
 
     if (!is_inter_block(edge_mbmi) ||
-        (!CHECK_LAST3_OR_LAST4(edge_mbmi->ref_frame[0]) &&
+        (!CHECK_GOLDEN_OR_LAST3(edge_mbmi->ref_frame[0]) &&
          !has_second_ref(edge_mbmi)))
       pred_context = 2;
     else if (!has_second_ref(edge_mbmi))
@@ -1582,105 +1386,4 @@
   return pred_context;
 }
 
-#if CONFIG_BIDIR_PRED
-
-#define CHECK_BWDREF_OR_ALTREF(ref_frame) \
-  ((ref_frame == BWDREF_FRAME) || (ref_frame == ALTREF_FRAME))
-// For the bit to signal whether the single reference is ALTREF_FRAME or
-// BWDREF_FRAME, knowing that it shall be either of these 2 choices.
-//
-// NOTE(zoeliu): The probability of ref_frame[0] is ALTREF_FRAME, conditioning
-// on it is either ALTREF_FRAME or BWDREF_FRAME.
-int vp10_get_pred_context_single_ref_p3(const MACROBLOCKD *xd) {
-  int pred_context;
-  const MB_MODE_INFO *const above_mbmi = xd->above_mbmi;
-  const MB_MODE_INFO *const left_mbmi = xd->left_mbmi;
-  const int has_above = xd->up_available;
-  const int has_left = xd->left_available;
-
-  // Note:
-  // The mode info data structure has a one element border above and to the
-  // left of the entries correpsonding to real macroblocks.
-  // The prediction flags in these dummy entries are initialised to 0.
-  if (has_above && has_left) {  // both edges available
-    const int above_intra = !is_inter_block(above_mbmi);
-    const int left_intra = !is_inter_block(left_mbmi);
-
-    if (above_intra && left_intra) {  // intra/intra
-      pred_context = 2;
-    } else if (above_intra || left_intra) {  // intra/inter or inter/intra
-      const MB_MODE_INFO *edge_mbmi = above_intra ? left_mbmi : above_mbmi;
-      if (!has_second_ref(edge_mbmi)) {
-        if (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]))
-          pred_context = 3;
-        else
-          pred_context = 4 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME);
-      } else {
-        pred_context = 1 +
-            2 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME ||
-                 edge_mbmi->ref_frame[1] == BWDREF_FRAME);
-      }
-    } else {  // inter/inter
-      const int above_has_second = has_second_ref(above_mbmi);
-      const int left_has_second  = has_second_ref(left_mbmi);
-      const MV_REFERENCE_FRAME above0 = above_mbmi->ref_frame[0];
-      const MV_REFERENCE_FRAME above1 = above_mbmi->ref_frame[1];
-      const MV_REFERENCE_FRAME left0 = left_mbmi->ref_frame[0];
-      const MV_REFERENCE_FRAME left1 = left_mbmi->ref_frame[1];
-
-      if (above_has_second && left_has_second) {
-        if (above0 == left0 && above1 == left1)
-          pred_context =
-              3 * (above0 == BWDREF_FRAME || above1 == BWDREF_FRAME ||
-                   left0 == BWDREF_FRAME || left1 == BWDREF_FRAME);
-        else
-          pred_context = 2;
-      } else if (above_has_second || left_has_second) {
-        const MV_REFERENCE_FRAME srf = !above_has_second ? above0 : left0;
-        const MV_REFERENCE_FRAME crf0 = above_has_second ? above0 : left0;
-        const MV_REFERENCE_FRAME crf1 = above_has_second ? above1 : left1;
-
-        if (srf == BWDREF_FRAME)
-          pred_context = 3 + (crf0 == BWDREF_FRAME || crf0 == BWDREF_FRAME);
-        else if (srf == ALTREF_FRAME)
-          pred_context = (crf0 == BWDREF_FRAME || crf1 == BWDREF_FRAME);
-        else
-          pred_context = 1 + 2 * (crf0 == BWDREF_FRAME || crf1 == BWDREF_FRAME);
-      } else {
-        if (!CHECK_BWDREF_OR_ALTREF(above0) &&
-            !CHECK_BWDREF_OR_ALTREF(left0)) {
-          pred_context = 2 + (above0 == left0);
-        } else if (!CHECK_BWDREF_OR_ALTREF(above0) ||
-                   !CHECK_BWDREF_OR_ALTREF(left0)) {
-          const MV_REFERENCE_FRAME edge0 =
-              !CHECK_BWDREF_OR_ALTREF(above0) ? left0 : above0;
-          pred_context = 4 * (edge0 == BWDREF_FRAME);
-        } else {
-          pred_context =
-              2 * (above0 == BWDREF_FRAME) + 2 * (left0 == BWDREF_FRAME);
-        }
-      }
-    }
-  } else if (has_above || has_left) {  // one edge available
-    const MB_MODE_INFO *edge_mbmi = has_above ? above_mbmi : left_mbmi;
-
-    if (!is_inter_block(edge_mbmi) ||
-        (!CHECK_BWDREF_OR_ALTREF(edge_mbmi->ref_frame[0]) &&
-         !has_second_ref(edge_mbmi)))
-      pred_context = 2;
-    else if (!has_second_ref(edge_mbmi))
-      pred_context = 4 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME);
-    else
-      pred_context = 3 * (edge_mbmi->ref_frame[0] == BWDREF_FRAME ||
-                          edge_mbmi->ref_frame[1] == BWDREF_FRAME);
-  } else {  // no edges available (2)
-    pred_context = 2;
-  }
-
-  assert(pred_context >= 0 && pred_context < REF_CONTEXTS);
-  return pred_context;
-}
-
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
diff --git a/vp10/common/pred_common.h b/vp10/common/pred_common.h
index 1c3c721..d4ae980 100644
--- a/vp10/common/pred_common.h
+++ b/vp10/common/pred_common.h
@@ -119,18 +119,6 @@
   return cm->fc->comp_ref_prob[pred_context][2];
 }
 
-int vp10_get_pred_context_comp_ref_p3(const VP10_COMMON *cm,
-                                      const MACROBLOCKD *xd);
-
-static INLINE vpx_prob vp10_get_pred_prob_comp_ref_p3(const VP10_COMMON *cm,
-                                                     const MACROBLOCKD *xd) {
-  const int pred_context = vp10_get_pred_context_comp_ref_p3(cm, xd);
-  return cm->fc->comp_ref_prob[pred_context][3];
-}
-
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
 int vp10_get_pred_context_comp_bwdref_p(const VP10_COMMON *cm,
                                         const MACROBLOCKD *xd);
 
@@ -139,7 +127,6 @@
   const int pred_context = vp10_get_pred_context_comp_bwdref_p(cm, xd);
   return cm->fc->comp_bwdref_prob[pred_context][0];
 }
-#endif  // CONFIG_BIDIR_PRED
 
 #endif  // CONFIG_EXT_REFS
 
@@ -157,16 +144,14 @@
   return cm->fc->single_ref_prob[vp10_get_pred_context_single_ref_p2(xd)][1];
 }
 
-#if CONFIG_EXT_REFS || CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 int vp10_get_pred_context_single_ref_p3(const MACROBLOCKD *xd);
 
 static INLINE vpx_prob vp10_get_pred_prob_single_ref_p3(const VP10_COMMON *cm,
                                                         const MACROBLOCKD *xd) {
   return cm->fc->single_ref_prob[vp10_get_pred_context_single_ref_p3(xd)][2];
 }
-#endif  // CONFIG_EXT_REFS || CONFIR_BIDIR_PRED
 
-#if CONFIG_EXT_REFS
 int vp10_get_pred_context_single_ref_p4(const MACROBLOCKD *xd);
 
 static INLINE vpx_prob vp10_get_pred_prob_single_ref_p4(const VP10_COMMON *cm,
diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c
index 6a51fbb..5d0eb47 100644
--- a/vp10/decoder/decodeframe.c
+++ b/vp10/decoder/decodeframe.c
@@ -57,43 +57,31 @@
 }
 
 static void setup_compound_reference_mode(VP10_COMMON *cm) {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   cm->comp_fwd_ref[0] = LAST_FRAME;
-  cm->comp_fwd_ref[1] = GOLDEN_FRAME;
+  cm->comp_fwd_ref[1] = LAST2_FRAME;
+  cm->comp_fwd_ref[2] = LAST3_FRAME;
+  cm->comp_fwd_ref[3] = GOLDEN_FRAME;
+
   cm->comp_bwd_ref[0] = BWDREF_FRAME;
   cm->comp_bwd_ref[1] = ALTREF_FRAME;
-
-#else  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-
+#else
   if (cm->ref_frame_sign_bias[LAST_FRAME] ==
           cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
     cm->comp_fixed_ref = ALTREF_FRAME;
     cm->comp_var_ref[0] = LAST_FRAME;
-#if CONFIG_EXT_REFS
-    cm->comp_var_ref[1] = LAST2_FRAME;
-    cm->comp_var_ref[2] = LAST3_FRAME;
-    cm->comp_var_ref[3] = LAST4_FRAME;
-    cm->comp_var_ref[4] = GOLDEN_FRAME;
-#else  // CONFIG_EXT_REFS
     cm->comp_var_ref[1] = GOLDEN_FRAME;
-#endif  // CONFIG_EXT_REFS
   } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
                  cm->ref_frame_sign_bias[ALTREF_FRAME]) {
-#if CONFIG_EXT_REFS
-    assert(0);
-#endif  // CONFIG_EXT_REFS
     cm->comp_fixed_ref = GOLDEN_FRAME;
     cm->comp_var_ref[0] = LAST_FRAME;
     cm->comp_var_ref[1] = ALTREF_FRAME;
   } else {
-#if CONFIG_EXT_REFS
-    assert(0);
-#endif  // CONFIG_EXT_REFS
     cm->comp_fixed_ref = LAST_FRAME;
     cm->comp_var_ref[0] = GOLDEN_FRAME;
     cm->comp_var_ref[1] = ALTREF_FRAME;
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 }
 
 static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
@@ -180,7 +168,7 @@
 
   if (cm->reference_mode != SINGLE_REFERENCE) {
     for (i = 0; i < REF_CONTEXTS; ++i) {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       for (j = 0; j < (FWD_REFS - 1); ++j)
         vp10_diff_update_prob(r, &fc->comp_ref_prob[i][j]);
       for (j = 0; j < (BWD_REFS - 1); ++j)
@@ -188,7 +176,7 @@
 #else
       for (j = 0; j < (COMP_REFS - 1); ++j)
         vp10_diff_update_prob(r, &fc->comp_ref_prob[i][j]);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     }
   }
 }
@@ -3128,10 +3116,10 @@
   cm->last_frame_type = cm->frame_type;
   cm->last_intra_only = cm->intra_only;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // NOTE: By default all coded frames to be used as a reference
   cm->is_reference_frame = 1;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   if (vpx_rb_read_literal(rb, 2) != VP9_FRAME_MARKER)
       vpx_internal_error(&cm->error, VPX_CODEC_UNSUP_BITSTREAM,
@@ -3167,7 +3155,7 @@
     cm->lf.filter_level = 0;
     cm->show_frame = 1;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     // NOTE(zoeliu): The existing frame to show is adopted as a reference frame.
     pbi->refresh_frame_flags = vpx_rb_read_literal(rb, REF_FRAMES);
 
@@ -3225,7 +3213,7 @@
       for (i = 0; i < REF_FRAMES; ++i)
         cm->next_ref_frame_map[i] = cm->ref_frame_map[i];
     }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
     return 0;
   }
@@ -3291,13 +3279,13 @@
     } else if (pbi->need_resync != 1) {  /* Skip if need resync */
       pbi->refresh_frame_flags = vpx_rb_read_literal(rb, REF_FRAMES);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       if (!pbi->refresh_frame_flags) {
         // NOTE: "pbi->refresh_frame_flags == 0" indicates that the coded frame
         //       will not be used as a reference
         cm->is_reference_frame = 0;
       }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
       for (i = 0; i < REFS_PER_FRAME; ++i) {
         const int ref = vpx_rb_read_literal(rb, REF_FRAMES_LOG2);
@@ -3692,12 +3680,12 @@
                  sizeof(cm->counts.single_ref)));
   assert(!memcmp(cm->counts.comp_ref, zero_counts.comp_ref,
                  sizeof(cm->counts.comp_ref)));
-  assert(!memcmp(&cm->counts.tx_size, &zero_counts.tx_size,
-                 sizeof(cm->counts.tx_size)));
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   assert(!memcmp(cm->counts.comp_bwdref, zero_counts.comp_bwdref,
                  sizeof(cm->counts.comp_bwdref)));
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
+  assert(!memcmp(&cm->counts.tx_size, &zero_counts.tx_size,
+                 sizeof(cm->counts.tx_size)));
   assert(!memcmp(cm->counts.skip, zero_counts.skip, sizeof(cm->counts.skip)));
 #if CONFIG_REF_MV
   assert(!memcmp(&cm->counts.mv[0], &zero_counts.mv[0],
@@ -3772,11 +3760,11 @@
 
   if (!first_partition_size) {
     // showing a frame directly
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     if (cm->show_existing_frame)
       *p_data_end = data + vpx_rb_bytes_read(&rb);
     else
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       *p_data_end = data + (cm->profile <= PROFILE_2 ? 1 : 2);
 
     return;
@@ -3793,7 +3781,7 @@
                            !cm->last_intra_only &&
                            cm->last_show_frame &&
                            (cm->last_frame_type != KEY_FRAME);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // NOTE(zoeliu): As cm->prev_frame can take neither a frame of
   //               show_exisiting_frame=1, nor can it take a frame not used as
   //               a reference, it is probable that by the time it is being
@@ -3809,7 +3797,7 @@
     RefBuffer *last_fb_ref_buf = &cm->frame_refs[LAST_FRAME - LAST_FRAME];
     cm->prev_frame = &cm->buffer_pool->frame_bufs[last_fb_ref_buf->idx];
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   vp10_setup_block_planes(xd, cm->subsampling_x, cm->subsampling_y);
 
diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 4a669b1..8528370 100644
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -766,68 +766,58 @@
     const REFERENCE_MODE mode = read_block_reference_mode(cm, xd, r);
     // FIXME(rbultje) I'm pretty sure this breaks segmentation ref frame coding
     if (mode == COMPOUND_REFERENCE) {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       const int idx = cm->ref_frame_sign_bias[cm->comp_bwd_ref[0]];
 #else
       const int idx = cm->ref_frame_sign_bias[cm->comp_fixed_ref];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       const int ctx = vp10_get_pred_context_comp_ref_p(cm, xd);
       const int bit = vp10_read(r, fc->comp_ref_prob[ctx][0]);
+
       if (counts)
         ++counts->comp_ref[ctx][0][bit];
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-      ref_frame[!idx] = cm->comp_fwd_ref[bit];
-      {
-        const int ctx1 = vp10_get_pred_context_comp_bwdref_p(cm, xd);
-        const int bit1 = vp10_read(r, fc->comp_bwdref_prob[ctx1][0]);
-        if (counts)
-          ++counts->comp_bwdref[ctx1][0][bit1];
-        ref_frame[idx] = cm->comp_bwd_ref[bit1];
-      }
-
-#else  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-
-      ref_frame[idx] = cm->comp_fixed_ref;
-
 #if CONFIG_EXT_REFS
+      // Decode forward references.
       if (!bit) {
         const int ctx1 = vp10_get_pred_context_comp_ref_p1(cm, xd);
         const int bit1 = vp10_read(r, fc->comp_ref_prob[ctx1][1]);
         if (counts)
           ++counts->comp_ref[ctx1][1][bit1];
-        ref_frame[!idx] = cm->comp_var_ref[bit1 ? 0 : 1];
+        ref_frame[!idx] = cm->comp_fwd_ref[bit1 ? 0 : 1];
       } else {
         const int ctx2 = vp10_get_pred_context_comp_ref_p2(cm, xd);
         const int bit2 = vp10_read(r, fc->comp_ref_prob[ctx2][2]);
         if (counts)
           ++counts->comp_ref[ctx2][2][bit2];
-        if (!bit2) {
-          const int ctx3 = vp10_get_pred_context_comp_ref_p3(cm, xd);
-          const int bit3 = vp10_read(r, fc->comp_ref_prob[ctx3][3]);
-          if (counts)
-            ++counts->comp_ref[ctx3][3][bit3];
-          ref_frame[!idx] = cm->comp_var_ref[bit3 ? 2 : 3];
-        } else {
-          ref_frame[!idx] = cm->comp_var_ref[4];
-        }
+        ref_frame[!idx] = cm->comp_fwd_ref[bit2 ? 3 : 2];
       }
-#else  // CONFIG_EXT_REFS
+
+      // Decode backward references.
+      {
+        const int ctx_bwd = vp10_get_pred_context_comp_bwdref_p(cm, xd);
+        const int bit_bwd = vp10_read(r, fc->comp_bwdref_prob[ctx_bwd][0]);
+        if (counts)
+          ++counts->comp_bwdref[ctx_bwd][0][bit_bwd];
+        ref_frame[idx] = cm->comp_bwd_ref[bit_bwd];
+      }
+#else
       ref_frame[!idx] = cm->comp_var_ref[bit];
+      ref_frame[idx] = cm->comp_fixed_ref;
 #endif  // CONFIG_EXT_REFS
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
     } else if (mode == SINGLE_REFERENCE) {
 #if CONFIG_EXT_REFS
       const int ctx0 = vp10_get_pred_context_single_ref_p1(xd);
       const int bit0 = vp10_read(r, fc->single_ref_prob[ctx0][0]);
       if (counts)
         ++counts->single_ref[ctx0][0][bit0];
+
       if (bit0) {
         const int ctx1 = vp10_get_pred_context_single_ref_p2(xd);
         const int bit1 = vp10_read(r, fc->single_ref_prob[ctx1][1]);
         if (counts)
           ++counts->single_ref[ctx1][1][bit1];
-        ref_frame[0] = bit1 ? ALTREF_FRAME : GOLDEN_FRAME;
+        ref_frame[0] = bit1 ? ALTREF_FRAME : BWDREF_FRAME;
       } else {
         const int ctx2 = vp10_get_pred_context_single_ref_p3(xd);
         const int bit2 = vp10_read(r, fc->single_ref_prob[ctx2][2]);
@@ -838,7 +828,7 @@
           const int bit4 = vp10_read(r, fc->single_ref_prob[ctx4][4]);
           if (counts)
             ++counts->single_ref[ctx4][4][bit4];
-          ref_frame[0] = bit4 ? LAST4_FRAME : LAST3_FRAME;
+          ref_frame[0] = bit4 ? GOLDEN_FRAME : LAST3_FRAME;
         } else {
           const int ctx3 = vp10_get_pred_context_single_ref_p4(xd);
           const int bit3 = vp10_read(r, fc->single_ref_prob[ctx3][3]);
@@ -847,29 +837,18 @@
           ref_frame[0] = bit3 ? LAST2_FRAME : LAST_FRAME;
         }
       }
-#else  // CONFIG_EXT_REFS
+#else
       const int ctx0 = vp10_get_pred_context_single_ref_p1(xd);
       const int bit0 = vp10_read(r, fc->single_ref_prob[ctx0][0]);
       if (counts)
         ++counts->single_ref[ctx0][0][bit0];
+
       if (bit0) {
         const int ctx1 = vp10_get_pred_context_single_ref_p2(xd);
         const int bit1 = vp10_read(r, fc->single_ref_prob[ctx1][1]);
         if (counts)
           ++counts->single_ref[ctx1][1][bit1];
-#if CONFIG_BIDIR_PRED
-        if (bit1) {
-          const int ctx2 = vp10_get_pred_context_single_ref_p3(xd);
-          const int bit2 = vp10_read(r, fc->single_ref_prob[ctx2][2]);
-          if (counts)
-            ++counts->single_ref[ctx2][2][bit2];
-          ref_frame[0] = bit2 ? ALTREF_FRAME : BWDREF_FRAME;
-        } else {
-          ref_frame[0] = GOLDEN_FRAME;
-        }
-#else  // CONFIG_BIDIR_PRED
         ref_frame[0] = bit1 ? ALTREF_FRAME : GOLDEN_FRAME;
-#endif  // CONFIG_BIDIR_PRED
       } else {
         ref_frame[0] = LAST_FRAME;
       }
diff --git a/vp10/decoder/decoder.c b/vp10/decoder/decoder.c
index 573266e..13e2261 100644
--- a/vp10/decoder/decoder.c
+++ b/vp10/decoder/decoder.c
@@ -208,12 +208,13 @@
   // #if CONFIG_EXT_REFS
   //   cpi->lst2_fb_idx = 1;
   //   cpi->lst3_fb_idx = 2;
-  //   cpi->lst4_fb_idx = 3;
-  //   cpi->gld_fb_idx = 4;
+  //   cpi->gld_fb_idx = 3;
+  //   cpi->bwd_fb_idx = 4;
   //   cpi->alt_fb_idx = 5;
   // #else  // CONFIG_EXT_REFS
   //   cpi->gld_fb_idx = 1;
   //   cpi->alt_fb_idx = 2;
+  // #endif  // CONFIG_EXT_REFS
 
   // TODO(zoeliu): To revisit following code and reconsider what assumption we
   // may take on the reference frame buffer virtual indexes
@@ -224,24 +225,17 @@
     idx = cm->ref_frame_map[1];
   } else if (ref_frame_flag == VP9_LAST3_FLAG) {
     idx = cm->ref_frame_map[2];
-  } else if (ref_frame_flag == VP9_LAST4_FLAG) {
-    idx = cm->ref_frame_map[3];
   } else if (ref_frame_flag == VP9_GOLD_FLAG) {
+    idx = cm->ref_frame_map[3];
+  } else if (ref_frame_flag == VP9_BWD_FLAG) {
     idx = cm->ref_frame_map[4];
   } else if (ref_frame_flag == VP9_ALT_FLAG) {
     idx = cm->ref_frame_map[5];
-#else  // CONFIG_EXT_REFS
+#else
   } else if (ref_frame_flag == VP9_GOLD_FLAG) {
     idx = cm->ref_frame_map[1];
-#if CONFIG_BIDIR_PRED
-  } else if (ref_frame_flag == VP9_BWD_FLAG) {
-    idx = cm->ref_frame_map[2];
-  } else if (ref_frame_flag == VP9_ALT_FLAG) {
-    idx = cm->ref_frame_map[3];
-#else  // CONFIG_BIDIR_PRED
   } else if (ref_frame_flag == VP9_ALT_FLAG) {
     idx = cm->ref_frame_map[2];
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   } else {
     vpx_internal_error(&cm->error, VPX_CODEC_ERROR,
@@ -291,7 +285,7 @@
   }
 
   // Current thread releases the holding of reference frame.
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   for (; ref_index < REF_FRAMES; ++ref_index) {
     const int old_idx = cm->ref_frame_map[ref_index];
     decrease_ref_count(old_idx, frame_bufs, pool);
@@ -303,7 +297,7 @@
     decrease_ref_count(old_idx, frame_bufs, pool);
     cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index];
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   unlock_buffer_pool(pool);
   pbi->hold_ref_buf = 0;
   cm->frame_to_show = get_frame_new_buffer(cm);
@@ -412,7 +406,7 @@
       }
 
       // Current thread releases the holding of reference frame.
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       for (; ref_index < REF_FRAMES; ++ref_index) {
         const int old_idx = cm->ref_frame_map[ref_index];
         decrease_ref_count(old_idx, frame_bufs, pool);
@@ -422,7 +416,7 @@
         const int old_idx = cm->ref_frame_map[ref_index];
         decrease_ref_count(old_idx, frame_bufs, pool);
       }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       pbi->hold_ref_buf = 0;
     }
     // Release current frame.
@@ -451,10 +445,10 @@
   if (!cm->show_existing_frame) {
     cm->last_show_frame = cm->show_frame;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     // NOTE: It is not supposed to ref to any frame not used as reference
     if (cm->is_reference_frame)
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       cm->prev_frame = cm->cur_frame;
 
     if (cm->seg.enabled && !cm->frame_parallel_decode)
diff --git a/vp10/decoder/decoder.h b/vp10/decoder/decoder.h
index 7991ac1..e4be2f4 100644
--- a/vp10/decoder/decoder.h
+++ b/vp10/decoder/decoder.h
@@ -152,7 +152,7 @@
   }
 }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 static INLINE int dec_is_ref_frame_buf(VP10Decoder *const pbi,
                                        RefCntBuffer *frame_buf) {
   VP10_COMMON *const cm = &pbi->common;
@@ -165,7 +165,7 @@
   }
   return (i < REFS_PER_FRAME);
 }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/vp10/decoder/dthread.c b/vp10/decoder/dthread.c
index a4555c8..f0cb98d 100644
--- a/vp10/decoder/dthread.c
+++ b/vp10/decoder/dthread.c
@@ -159,10 +159,9 @@
 #if CONFIG_VP9_HIGHBITDEPTH
   dst_cm->use_highbitdepth = src_cm->use_highbitdepth;
 #endif
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // TODO(zoeliu): To handle parallel decoding
-  assert(0);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   dst_cm->prev_frame = src_cm->show_existing_frame ?
                        src_cm->prev_frame : src_cm->cur_frame;
   dst_cm->last_width = !src_cm->show_existing_frame ?
diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index 97b42d3..7557ce4 100644
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -839,14 +839,12 @@
     if (is_compound) {
 #if CONFIG_EXT_REFS
       const int bit = (mbmi->ref_frame[0] == GOLDEN_FRAME ||
-                       mbmi->ref_frame[0] == LAST3_FRAME ||
-                       mbmi->ref_frame[0] == LAST4_FRAME);
+                       mbmi->ref_frame[0] == LAST3_FRAME);
+      const int bit_bwd = mbmi->ref_frame[1] == ALTREF_FRAME;
 #else  // CONFIG_EXT_REFS
       const int bit = mbmi->ref_frame[0] == GOLDEN_FRAME;
-#if CONFIG_BIDIR_PRED
-      const int bit_bwd = mbmi->ref_frame[1] == ALTREF_FRAME;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
+
       vp10_write(w, bit, vp10_get_pred_prob_comp_ref_p(cm, xd));
 
 #if CONFIG_EXT_REFS
@@ -856,28 +854,21 @@
       } else {
         const int bit2 = mbmi->ref_frame[0] == GOLDEN_FRAME;
         vp10_write(w, bit2, vp10_get_pred_prob_comp_ref_p2(cm, xd));
-        if (!bit2) {
-          const int bit3 = mbmi->ref_frame[0] == LAST3_FRAME;
-          vp10_write(w, bit3, vp10_get_pred_prob_comp_ref_p3(cm, xd));
-        }
       }
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
       vp10_write(w, bit_bwd, vp10_get_pred_prob_comp_bwdref_p(cm, xd));
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
     } else {
 #if CONFIG_EXT_REFS
-      const int bit0 = (mbmi->ref_frame[0] == GOLDEN_FRAME ||
-                        mbmi->ref_frame[0] == ALTREF_FRAME);
+      const int bit0 = (mbmi->ref_frame[0] == ALTREF_FRAME ||
+                        mbmi->ref_frame[0] == BWDREF_FRAME);
       vp10_write(w, bit0, vp10_get_pred_prob_single_ref_p1(cm, xd));
 
       if (bit0) {
-        const int bit1 = mbmi->ref_frame[0] != GOLDEN_FRAME;
+        const int bit1 = mbmi->ref_frame[0] == ALTREF_FRAME;
         vp10_write(w, bit1, vp10_get_pred_prob_single_ref_p2(cm, xd));
       } else {
         const int bit2 = (mbmi->ref_frame[0] == LAST3_FRAME ||
-                          mbmi->ref_frame[0] == LAST4_FRAME);
+                          mbmi->ref_frame[0] == GOLDEN_FRAME);
         vp10_write(w, bit2, vp10_get_pred_prob_single_ref_p3(cm, xd));
 
         if (!bit2) {
@@ -891,15 +882,10 @@
 #else  // CONFIG_EXT_REFS
       const int bit0 = mbmi->ref_frame[0] != LAST_FRAME;
       vp10_write(w, bit0, vp10_get_pred_prob_single_ref_p1(cm, xd));
+
       if (bit0) {
         const int bit1 = mbmi->ref_frame[0] != GOLDEN_FRAME;
         vp10_write(w, bit1, vp10_get_pred_prob_single_ref_p2(cm, xd));
-#if CONFIG_BIDIR_PRED
-        if (bit1) {
-          const int bit2 = mbmi->ref_frame[0] != BWDREF_FRAME;
-          vp10_write(w, bit2, vp10_get_pred_prob_single_ref_p3(cm, xd));
-        }
-#endif  // CONFIG_BIDIR_PRED
       }
 #endif  // CONFIG_EXT_REFS
     }
@@ -2681,21 +2667,19 @@
 
 #if CONFIG_EXT_REFS
   // NOTE(zoeliu): When LAST_FRAME is to get refreshed, the decoder will be
-  // notified to get LAST4_FRAME refreshed and then the virtual indexes for all
-  // the 4 LAST reference frames will be updated accordingly, i.e.:
-  // (1) The original virtual index for LAST4_FRAME will become the new virtual
+  // notified to get LAST3_FRAME refreshed and then the virtual indexes for all
+  // the 3 LAST reference frames will be updated accordingly, i.e.:
+  // (1) The original virtual index for LAST3_FRAME will become the new virtual
   //     index for LAST_FRAME; and
-  // (2) The original virtual indexes for LAST_FRAME ~ LAST3_FRAME will be
-  //     shifted and become the new virtual indexes for LAST2_FRAME ~
-  //     LAST4_FRAME.
-  refresh_mask |= (cpi->refresh_last_frame <<
-      cpi->lst_fb_idxes[LAST4_FRAME - LAST_FRAME]);
-#else  // CONFIG_EXT_REFS
-  refresh_mask |= (cpi->refresh_last_frame << cpi->lst_fb_idx);
+  // (2) The original virtual indexes for LAST_FRAME and LAST2_FRAME will be
+  //     shifted and become the new virtual indexes for LAST2_FRAME and
+  //     LAST3_FRAME.
+  refresh_mask |=
+      (cpi->refresh_last_frame << cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]);
 
-#if CONFIG_BIDIR_PRED
   refresh_mask |= (cpi->refresh_bwd_ref_frame << cpi->bwd_fb_idx);
-#endif  // CONFIG_BIDIR_PRED
+#else
+  refresh_mask |= (cpi->refresh_last_frame << cpi->lst_fb_idx);
 #endif  // CONFIG_EXT_REFS
 
   if (vp10_preserve_existing_gf(cpi)) {
@@ -3048,11 +3032,12 @@
 
   write_profile(cm->profile, wb);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // NOTE: By default all coded frames to be used as a reference
   cm->is_reference_frame = 1;
 
   if (cm->show_existing_frame) {
+    MV_REFERENCE_FRAME ref_frame;
     RefCntBuffer *const frame_bufs = cm->buffer_pool->frame_bufs;
     const int frame_to_show =
         cm->ref_frame_map[cpi->existing_fb_idx_to_show];
@@ -3069,23 +3054,21 @@
 
     cpi->refresh_frame_mask = get_refresh_mask(cpi);
     vpx_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
-    {
-      MV_REFERENCE_FRAME ref_frame;
-      for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
-        assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
-        vpx_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame),
-                             REF_FRAMES_LOG2);
-        vpx_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]);
-      }
+
+    for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
+      assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
+      vpx_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame),
+                           REF_FRAMES_LOG2);
+      vpx_wb_write_bit(wb, cm->ref_frame_sign_bias[ref_frame]);
     }
 
     return;
   } else {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     vpx_wb_write_bit(wb, 0);  // show_existing_frame
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   vpx_wb_write_bit(wb, cm->frame_type);
   vpx_wb_write_bit(wb, cm->show_frame);
@@ -3114,36 +3097,36 @@
       }
     }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     cpi->refresh_frame_mask = get_refresh_mask(cpi);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
     if (cm->intra_only) {
       write_sync_code(wb);
       write_bitdepth_colorspace_sampling(cm, wb);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       vpx_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
 #else
       vpx_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       write_frame_size(cm, wb);
     } else {
       MV_REFERENCE_FRAME ref_frame;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       vpx_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES);
 #else
       vpx_wb_write_literal(wb, get_refresh_mask(cpi), REF_FRAMES);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       if (!cpi->refresh_frame_mask) {
         // NOTE: "cpi->refresh_frame_mask == 0" indicates that the coded frame
         //       will not be used as a reference
         cm->is_reference_frame = 0;
       }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
       for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
         assert(get_ref_frame_map_idx(cpi, ref_frame) != INVALID_IDX);
@@ -3333,7 +3316,7 @@
 
     if (cm->reference_mode != SINGLE_REFERENCE) {
       for (i = 0; i < REF_CONTEXTS; i++) {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
         for (j = 0; j < (FWD_REFS - 1); j++) {
           vp10_cond_prob_diff_update(header_bc, &fc->comp_ref_prob[i][j],
                                      counts->comp_ref[i][j]);
@@ -3342,12 +3325,12 @@
           vp10_cond_prob_diff_update(header_bc, &fc->comp_bwdref_prob[i][j],
                                      counts->comp_bwdref[i][j]);
         }
-#else  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#else
         for (j = 0; j < (COMP_REFS - 1); j++) {
           vp10_cond_prob_diff_update(header_bc, &fc->comp_ref_prob[i][j],
                                      counts->comp_ref[i][j]);
         }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       }
     }
 
@@ -3541,12 +3524,12 @@
   // Write the uncompressed header
   write_uncompressed_header(cpi, &wb);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (cm->show_existing_frame) {
     *size = vpx_wb_bytes_written(&wb);
     return;
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // We do not know these in advance. Output placeholder bit.
   saved_wb = wb;
diff --git a/vp10/encoder/denoiser.c b/vp10/encoder/denoiser.c
index 5a6ae4a..1ff854d 100644
--- a/vp10/encoder/denoiser.c
+++ b/vp10/encoder/denoiser.c
@@ -388,9 +388,9 @@
                                     YV12_BUFFER_CONFIG src,
                                     FRAME_TYPE frame_type,
                                     int refresh_last_frame,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
                                     int refresh_bwd_ref_frame,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
                                     int refresh_alt_ref_frame,
                                     int refresh_golden_frame) {
   if (frame_type == KEY_FRAME) {
@@ -410,16 +410,18 @@
     swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
                       &denoiser->running_avg_y[INTRA_FRAME]);
   }
+  // TODO(zoeliu): To explore whether when show_existing_frame == 1 should be
+  //               handled differently.
   if (refresh_last_frame) {
     swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
                       &denoiser->running_avg_y[INTRA_FRAME]);
   }
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (refresh_bwd_ref_frame) {
     swap_frame_buffer(&denoiser->running_avg_y[BWDREF_FRAME],
                       &denoiser->running_avg_y[INTRA_FRAME]);
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 }
 
 void vp10_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
diff --git a/vp10/encoder/denoiser.h b/vp10/encoder/denoiser.h
index ceef451..d7fdf0c 100644
--- a/vp10/encoder/denoiser.h
+++ b/vp10/encoder/denoiser.h
@@ -36,9 +36,9 @@
                                     YV12_BUFFER_CONFIG src,
                                     FRAME_TYPE frame_type,
                                     int refresh_last_frame,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
                                     int refresh_bwd_ref_frame,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
                                     int refresh_alt_ref_frame,
                                     int refresh_golden_frame);
 
diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 014a874..683e2db 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -1909,17 +1909,18 @@
       // the reference frame counts used to work out probabilities.
       if (inter_block) {
         const MV_REFERENCE_FRAME ref0 = mbmi->ref_frame[0];
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
         const MV_REFERENCE_FRAME ref1 = mbmi->ref_frame[1];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
+
         if (cm->reference_mode == REFERENCE_MODE_SELECT)
           counts->comp_inter[vp10_get_reference_mode_context(cm, xd)]
                             [has_second_ref(mbmi)]++;
 
         if (has_second_ref(mbmi)) {
 #if CONFIG_EXT_REFS
-          const int bit = (ref0 == GOLDEN_FRAME || ref0 == LAST3_FRAME ||
-                           ref0 == LAST4_FRAME);
+          const int bit = (ref0 == GOLDEN_FRAME || ref0 == LAST3_FRAME);
+
           counts->comp_ref[vp10_get_pred_context_comp_ref_p(cm, xd)][0][bit]++;
           if (!bit) {
             counts->comp_ref[vp10_get_pred_context_comp_ref_p1(cm, xd)][1]
@@ -1927,26 +1928,22 @@
           } else {
             counts->comp_ref[vp10_get_pred_context_comp_ref_p2(cm, xd)][2]
                             [ref0 == GOLDEN_FRAME]++;
-            if (ref0 != GOLDEN_FRAME) {
-              counts->comp_ref[vp10_get_pred_context_comp_ref_p3(cm, xd)][3]
-                              [ref0 == LAST3_FRAME]++;
-            }
           }
-#else  // CONFIG_EXT_REFS
-          counts->comp_ref[vp10_get_pred_context_comp_ref_p(cm, xd)][0]
-                          [ref0 == GOLDEN_FRAME]++;
-#if CONFIG_BIDIR_PRED
+
           counts->comp_bwdref[vp10_get_pred_context_comp_bwdref_p(cm, xd)][0]
                              [ref1 == ALTREF_FRAME]++;
-#endif  // CONFIG_BIDIR_PRED
+#else
+          counts->comp_ref[vp10_get_pred_context_comp_ref_p(cm, xd)][0]
+                          [ref0 == GOLDEN_FRAME]++;
 #endif  // CONFIG_EXT_REFS
         } else {
 #if CONFIG_EXT_REFS
-          const int bit = (ref0 == ALTREF_FRAME || ref0 == GOLDEN_FRAME);
+          const int bit = (ref0 == ALTREF_FRAME || ref0 == BWDREF_FRAME);
+
           counts->single_ref[vp10_get_pred_context_single_ref_p1(xd)][0][bit]++;
           if (bit) {
             counts->single_ref[vp10_get_pred_context_single_ref_p2(xd)][1]
-                              [ref0 != GOLDEN_FRAME]++;
+                              [ref0 != BWDREF_FRAME]++;
           } else {
             const int bit1 = !(ref0 == LAST2_FRAME || ref0 == LAST_FRAME);
             counts->single_ref[vp10_get_pred_context_single_ref_p3(xd)][2]
@@ -1959,18 +1956,12 @@
                                 [ref0 != LAST3_FRAME]++;
             }
           }
-#else  // CONFIG_EXT_REFS
+#else
           counts->single_ref[vp10_get_pred_context_single_ref_p1(xd)][0]
                             [ref0 != LAST_FRAME]++;
           if (ref0 != LAST_FRAME) {
             counts->single_ref[vp10_get_pred_context_single_ref_p2(xd)][1]
                               [ref0 != GOLDEN_FRAME]++;
-#if CONFIG_BIDIR_PRED
-            if (ref0 != GOLDEN_FRAME) {
-              counts->single_ref[vp10_get_pred_context_single_ref_p3(xd)][2]
-                                [ref0 != BWDREF_FRAME]++;
-            }
-#endif  // CONFIG_BIDIR_PRED
           }
 #endif  // CONFIG_EXT_REFS
         }
@@ -4398,11 +4389,7 @@
 #if CONFIG_EXT_REFS
             !!(ref_flags & VP9_LAST2_FLAG) +
             !!(ref_flags & VP9_LAST3_FLAG) +
-            !!(ref_flags & VP9_LAST4_FLAG) +
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
             !!(ref_flags & VP9_BWD_FLAG) +
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
             !!(ref_flags & VP9_ALT_FLAG)) >= 2;
   }
@@ -4586,7 +4573,7 @@
                            cm->height == cm->last_height &&
                            !cm->intra_only &&
                            cm->last_show_frame;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // NOTE(zoeliu): As cm->prev_frame can take neither a frame of
   //               show_exisiting_frame=1, nor can it take a frame not used as
   //               a reference, it is probable that by the time it is being
@@ -4602,7 +4589,7 @@
     const int last_fb_buf_idx = get_ref_frame_buf_idx(cpi, LAST_FRAME);
     cm->prev_frame = &cm->buffer_pool->frame_bufs[last_fb_buf_idx];
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Special case: set prev_mi to NULL when the previous mode info
   // context cannot be used.
@@ -4669,23 +4656,18 @@
     } else {
       cpi->allow_comp_inter_inter = 1;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       cm->comp_fwd_ref[0] = LAST_FRAME;
-      cm->comp_fwd_ref[1] = GOLDEN_FRAME;
+      cm->comp_fwd_ref[1] = LAST2_FRAME;
+      cm->comp_fwd_ref[2] = LAST3_FRAME;
+      cm->comp_fwd_ref[3] = GOLDEN_FRAME;
       cm->comp_bwd_ref[0] = BWDREF_FRAME;
       cm->comp_bwd_ref[1] = ALTREF_FRAME;
-#else  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#else
       cm->comp_fixed_ref = ALTREF_FRAME;
       cm->comp_var_ref[0] = LAST_FRAME;
-#if CONFIG_EXT_REFS
-      cm->comp_var_ref[1] = LAST2_FRAME;
-      cm->comp_var_ref[2] = LAST3_FRAME;
-      cm->comp_var_ref[3] = LAST4_FRAME;
-      cm->comp_var_ref[4] = GOLDEN_FRAME;
-#else  // CONFIG_EXT_REFS
       cm->comp_var_ref[1] = GOLDEN_FRAME;
 #endif  // CONFIG_EXT_REFS
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
     }
   } else {
     cpi->allow_comp_inter_inter = 0;
diff --git a/vp10/encoder/encoder.c b/vp10/encoder/encoder.c
index 5748792..97a7299 100644
--- a/vp10/encoder/encoder.c
+++ b/vp10/encoder/encoder.c
@@ -901,16 +901,12 @@
   for (fb_idx = 0; fb_idx < LAST_REF_FRAMES; ++fb_idx)
     cpi->lst_fb_idxes[fb_idx] = fb_idx;
   cpi->gld_fb_idx = LAST_REF_FRAMES;
-  cpi->alt_fb_idx = cpi->gld_fb_idx + 1;
-#else  // CONFIG_EXT_REFS
+  cpi->bwd_fb_idx = LAST_REF_FRAMES + 1;
+  cpi->alt_fb_idx = LAST_REF_FRAMES + 2;
+#else
   cpi->lst_fb_idx = 0;
   cpi->gld_fb_idx = 1;
-#if CONFIG_BIDIR_PRED
-  cpi->bwd_fb_idx = 2;
-  cpi->alt_fb_idx = 3;
-#else  // CONFIG_BIDIR_PRED
   cpi->alt_fb_idx = 2;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 }
 
@@ -2235,11 +2231,11 @@
     rc->baseline_gf_interval = (MIN_GF_INTERVAL + MAX_GF_INTERVAL) / 2;
   }
 
-  cpi->refresh_golden_frame = 0;
   cpi->refresh_last_frame = 1;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+  cpi->refresh_golden_frame = 0;
+#if CONFIG_EXT_REFS
   cpi->refresh_bwd_ref_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   cm->refresh_frame_context =
       (oxcf->error_resilient_mode || oxcf->frame_parallel_decoding_mode) ?
@@ -2309,11 +2305,11 @@
   cpi->alt_ref_source = NULL;
   rc->is_src_frame_alt_ref = 0;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   rc->is_bwd_ref_frame = 0;
   rc->is_last_bipred_frame = 0;
   rc->is_bipred_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 #if 0
   // Experimental RD Code
@@ -2992,15 +2988,13 @@
     ref_frame = LAST2_FRAME;
   else if (ref_frame_flag == VP9_LAST3_FLAG)
     ref_frame = LAST3_FRAME;
-  else if (ref_frame_flag == VP9_LAST4_FLAG)
-    ref_frame = LAST4_FRAME;
 #endif  // CONFIG_EXT_REFS
   else if (ref_frame_flag == VP9_GOLD_FLAG)
     ref_frame = GOLDEN_FRAME;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   else if (ref_frame_flag == VP9_BWD_FLAG)
     ref_frame = BWDREF_FRAME;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   else if (ref_frame_flag == VP9_ALT_FLAG)
     ref_frame = ALTREF_FRAME;
 
@@ -3390,9 +3384,7 @@
   BufferPool *const pool = cm->buffer_pool;
   const int use_upsampled_ref = cpi->sf.use_upsampled_references;
   int new_uidx = 0;
-
 #if CONFIG_EXT_REFS
-  int ref_frame;
 #endif  // CONFIG_EXT_REFS
 
   // NOTE: Save the new show frame buffer index for --test-code=warn, i.e.,
@@ -3400,7 +3392,7 @@
   if (cm->show_frame)
     cpi->last_show_frame_buf_idx = cm->new_fb_idx;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // TODO(zoeliu): To remove the reference buffer update for the
   //               show_existing_frame==1 case.
 #if 0
@@ -3415,24 +3407,24 @@
     return;
   }
 #endif  // 0
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   if (use_upsampled_ref) {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     if (cm->show_existing_frame) {
       new_uidx = cpi->upsampled_ref_idx[cpi->existing_fb_idx_to_show];
       // TODO(zoeliu): Once following is confirmed, remove it.
       assert(cpi->upsampled_ref_bufs[new_uidx].ref_count > 0);
     } else {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       // Up-sample the current encoded frame.
       RefCntBuffer *bufs = pool->frame_bufs;
       const YV12_BUFFER_CONFIG *const ref = &bufs[cm->new_fb_idx].buf;
 
       new_uidx = upsample_ref_frame(cpi, ref);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   }
 
   // At this point the new frame has been encoded.
@@ -3440,20 +3432,20 @@
   if (cm->frame_type == KEY_FRAME) {
     ref_cnt_fb(pool->frame_bufs,
                &cm->ref_frame_map[cpi->gld_fb_idx], cm->new_fb_idx);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     ref_cnt_fb(pool->frame_bufs,
                &cm->ref_frame_map[cpi->bwd_fb_idx], cm->new_fb_idx);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     ref_cnt_fb(pool->frame_bufs,
                &cm->ref_frame_map[cpi->alt_fb_idx], cm->new_fb_idx);
 
     if (use_upsampled_ref) {
       uref_cnt_fb(cpi->upsampled_ref_bufs,
                   &cpi->upsampled_ref_idx[cpi->gld_fb_idx], new_uidx);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       uref_cnt_fb(cpi->upsampled_ref_bufs,
                   &cpi->upsampled_ref_idx[cpi->bwd_fb_idx], new_uidx);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       uref_cnt_fb(cpi->upsampled_ref_bufs,
                   &cpi->upsampled_ref_idx[cpi->alt_fb_idx], new_uidx);
     }
@@ -3516,7 +3508,7 @@
                sizeof(cpi->interp_filter_selected[ALTREF_FRAME]));
     }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     if (cpi->refresh_bwd_ref_frame) {
       ref_cnt_fb(pool->frame_bufs,
                  &cm->ref_frame_map[cpi->bwd_fb_idx], cm->new_fb_idx);
@@ -3528,7 +3520,7 @@
              cpi->interp_filter_selected[0],
              sizeof(cpi->interp_filter_selected[0]));
     }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   }
 
   if (cpi->refresh_last_frame) {
@@ -3537,110 +3529,94 @@
     // reference to the reference frame buffer virtual index; and then (2) from
     // the virtual index to the reference frame buffer physical index:
     //
-    // LAST_FRAME,      ..., LAST4_FRAME,     GOLDEN_FRAME,    ALTREF_FRAME
-    //      |                     |                |                |
-    //      v                     v                v                v
-    // lst_fb_idxes[0], ..., lst_fb_idxes[3], gld_fb_idx,      alt_fb_idx
-    //      |                     |                |                |
-    //      v                     v                v                v
-    // ref_frame_map[], ..., ref_frame_map[], ref_frame_map[], ref_frame_map[]
+    // LAST_FRAME,      ..., LAST3_FRAME,     ..., ALTREF_FRAME
+    //      |                     |                     |
+    //      v                     v                     v
+    // lst_fb_idxes[0], ..., lst_fb_idxes[2], ..., alt_fb_idx
+    //      |                     |                     |
+    //      v                     v                     v
+    // ref_frame_map[], ..., ref_frame_map[], ..., ref_frame_map[]
     //
-    // When refresh_last_frame is set, it is intended to retire LAST4_FRAME,
-    // have all the other 3 reference frames shifted as follows:
-    // LAST_FRAME -> LAST2_FRAME -> LAST3_FRAME -> LAST4_FRAME,
+    // When refresh_last_frame is set, it is intended to retire LAST3_FRAME,
+    // have the other 2 LAST reference frames shifted as follows:
+    // LAST_FRAME -> LAST2_FRAME -> LAST3_FRAME
     // , and then have LAST_FRAME refreshed by the newly coded frame.
     //
     // To fulfill it, the decoder will be notified to execute following 2 steps:
     //
-    // (a) To change ref_frame_map[] and have the virtual index of LAST4_FRAME
+    // (a) To change ref_frame_map[] and have the virtual index of LAST3_FRAME
     //     to point to the newly coded frame, i.e.
-    //     ref_frame_map[lst_fb_idexes[3]] => new_fb_idx;
+    //     ref_frame_map[lst_fb_idexes[2]] => new_fb_idx;
     //
     // (b) To change the 1st layer mapping to have LAST_FRAME mapped to the
-    //     original virtual index of LAST4_FRAME and have all the other mapping
+    //     original virtual index of LAST3_FRAME and have the other mappings
     //     shifted as follows:
-    // LAST_FRAME,      LAST2_FRAME,     LAST3_FRAME,     LAST4_FRAME
-    //      |                |                |                |
-    //      v                v                v                v
-    // lst_fb_idxes[3], lst_fb_idxes[0], lst_fb_idxes[1], lst_fb_idxes[2]
-    int tmp;
+    // LAST_FRAME,      LAST2_FRAME,     LAST3_FRAME
+    //      |                |                |
+    //      v                v                v
+    // lst_fb_idxes[2], lst_fb_idxes[0], lst_fb_idxes[1]
+    int ref_frame;
 
     if (cm->frame_type == KEY_FRAME) {
-      for (ref_frame = LAST_FRAME; ref_frame <= LAST4_FRAME; ++ref_frame) {
+      for (ref_frame = 0; ref_frame < LAST_REF_FRAMES; ++ref_frame) {
         ref_cnt_fb(pool->frame_bufs,
-                   &cm->ref_frame_map[cpi->lst_fb_idxes[ref_frame-LAST_FRAME]],
+                   &cm->ref_frame_map[cpi->lst_fb_idxes[ref_frame]],
                    cm->new_fb_idx);
 
         if (use_upsampled_ref)
           uref_cnt_fb(
               cpi->upsampled_ref_bufs,
-              &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[ref_frame-LAST_FRAME]],
+              &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[ref_frame]],
               new_uidx);
       }
     } else {
+      int tmp;
+
       ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->lst_fb_idxes[LAST4_FRAME-LAST_FRAME]],
+                 &cm->ref_frame_map[cpi->lst_fb_idxes[LAST_REF_FRAMES-1]],
                  cm->new_fb_idx);
 
       if (use_upsampled_ref)
         uref_cnt_fb(
             cpi->upsampled_ref_bufs,
-            &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[LAST4_FRAME-LAST_FRAME]],
+            &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[LAST_REF_FRAMES-1]],
             new_uidx);
 
-      tmp = cpi->lst_fb_idxes[LAST4_FRAME-LAST_FRAME];
-      for (ref_frame = LAST4_FRAME; ref_frame > LAST_FRAME; --ref_frame) {
-        cpi->lst_fb_idxes[ref_frame - LAST_FRAME] =
-            cpi->lst_fb_idxes[ref_frame - LAST_FRAME - 1];
+      tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES-1];
+      for (ref_frame = LAST_REF_FRAMES - 1; ref_frame > 0; --ref_frame) {
+        cpi->lst_fb_idxes[ref_frame] = cpi->lst_fb_idxes[ref_frame - 1];
+
         if (!cpi->rc.is_src_frame_alt_ref) {
           memcpy(cpi->interp_filter_selected[ref_frame],
                  cpi->interp_filter_selected[ref_frame - 1],
                  sizeof(cpi->interp_filter_selected[ref_frame - 1]));
         }
       }
-      cpi->lst_fb_idxes[LAST_FRAME-LAST_FRAME] = tmp;
+      cpi->lst_fb_idxes[0] = tmp;
+
+      if (!cpi->rc.is_src_frame_alt_ref) {
+        if (cm->show_existing_frame) {
+          memcpy(cpi->interp_filter_selected[LAST_FRAME],
+                 cpi->interp_filter_selected[BWDREF_FRAME],
+                 sizeof(cpi->interp_filter_selected[BWDREF_FRAME]));
+        } else {
+          memcpy(cpi->interp_filter_selected[LAST_FRAME],
+                 cpi->interp_filter_selected[0],
+                 sizeof(cpi->interp_filter_selected[0]));
+        }
+      }
+    }
+#else
+    ref_cnt_fb(pool->frame_bufs,
+               &cm->ref_frame_map[cpi->lst_fb_idx], cm->new_fb_idx);
+    if (use_upsampled_ref)
+      uref_cnt_fb(cpi->upsampled_ref_bufs,
+                  &cpi->upsampled_ref_idx[cpi->lst_fb_idx], new_uidx);
+    if (!cpi->rc.is_src_frame_alt_ref) {
       memcpy(cpi->interp_filter_selected[LAST_FRAME],
              cpi->interp_filter_selected[0],
              sizeof(cpi->interp_filter_selected[0]));
     }
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-    // TODO(zoeliu): To remove the reference buffer update for the
-    // show_existing_frame==1 case; Instead, we move the reference buffer update
-    // to the previous coded frame, i.e. the last-bipred-frame. In that case, no
-    // bit should be set in the refresh-mask, but the visual ref-idx should be
-    // updated and written to the bitstream accordingly, as the virtual ref-idx
-    // for LAST_FRAME and BWDREF_FRAME should be switched, i.e. cpi->lst_fb_idx
-    // and cpi->bwd_fb_idx should be switched.
-    if (cm->show_existing_frame) {
-      ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->lst_fb_idx], cm->new_fb_idx);
-
-      if (use_upsampled_ref)
-        uref_cnt_fb(cpi->upsampled_ref_bufs,
-                    &cpi->upsampled_ref_idx[cpi->lst_fb_idx], new_uidx);
-
-      // NOTE(zoeliu): OVERLAY should not be the last non-reference frame.
-      assert(!cpi->rc.is_src_frame_alt_ref);
-
-      memcpy(cpi->interp_filter_selected[LAST_FRAME],
-             cpi->interp_filter_selected[BWDREF_FRAME],
-             sizeof(cpi->interp_filter_selected[BWDREF_FRAME]));
-    } else {
-#endif  // CONFIG_BIDIR_PRED
-      ref_cnt_fb(pool->frame_bufs,
-                 &cm->ref_frame_map[cpi->lst_fb_idx], cm->new_fb_idx);
-      if (use_upsampled_ref)
-        uref_cnt_fb(cpi->upsampled_ref_bufs,
-                    &cpi->upsampled_ref_idx[cpi->lst_fb_idx], new_uidx);
-      if (!cpi->rc.is_src_frame_alt_ref) {
-        memcpy(cpi->interp_filter_selected[LAST_FRAME],
-               cpi->interp_filter_selected[0],
-               sizeof(cpi->interp_filter_selected[0]));
-      }
-#if CONFIG_BIDIR_PRED
-    }
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   }
 
@@ -3655,9 +3631,9 @@
                                    *cpi->Source,
                                    cpi->common.frame_type,
                                    cpi->refresh_last_frame,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
                                    cpi->refresh_bwd_ref_frame,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
                                    cpi->refresh_alt_ref_frame,
                                    cpi->refresh_golden_frame);
   }
@@ -3733,12 +3709,11 @@
 #if CONFIG_EXT_REFS
     VP9_LAST2_FLAG,
     VP9_LAST3_FLAG,
-    VP9_LAST4_FLAG,
 #endif  // CONFIG_EXT_REFS
     VP9_GOLD_FLAG,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     VP9_BWD_FLAG,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     VP9_ALT_FLAG
   };
 
@@ -3857,17 +3832,13 @@
     int refresh[REFS_PER_FRAME];
     refresh[0] = (cpi->refresh_last_frame) ? 1 : 0;
 #if CONFIG_EXT_REFS
-    refresh[1] = refresh[2] = refresh[3] = 0;
-    refresh[4] = (cpi->refresh_golden_frame) ? 1 : 0;
+    refresh[1] = refresh[2] = 0;
+    refresh[3] = (cpi->refresh_golden_frame) ? 1 : 0;
+    refresh[4] = (cpi->refresh_bwd_ref_frame) ? 1 : 0;
     refresh[5] = (cpi->refresh_alt_ref_frame) ? 1 : 0;
-#else  // CONFIG_EXT_REFS
+#else
     refresh[1] = (cpi->refresh_golden_frame) ? 1 : 0;
-#if CONFIG_BIDIR_PRED
-    refresh[2] = (cpi->refresh_bwd_ref_frame) ? 1 : 0;
-    refresh[3] = (cpi->refresh_alt_ref_frame) ? 1 : 0;
-#else  // CONFIG_BIDIR_PRED
     refresh[2] = (cpi->refresh_alt_ref_frame) ? 1 : 0;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
     for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
       const int idx = cpi->scaled_ref_idx[i - 1];
@@ -4002,12 +3973,7 @@
       // after a key/intra-only frame.
       cpi->max_mv_magnitude = max_mv_def;
     } else {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-      // TODO(zoeliu): Maybe we should leave it the same as base.
-      if (cm->show_frame || cpi->rc.is_bwd_ref_frame) {
-#else
       if (cm->show_frame) {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
         // Allow mv_steps to correspond to twice the max mv magnitude found
         // in the previous frame, capped by the default max_mv_magnitude based
         // on resolution.
@@ -4590,48 +4556,43 @@
   const int *const map = cpi->common.ref_frame_map;
 
 #if CONFIG_EXT_REFS
-  const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[0]];
-  const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[0]];
-
   const int last2_is_last =
       map[cpi->lst_fb_idxes[1]] == map[cpi->lst_fb_idxes[0]];
-  const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]];
-  const int last2_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
-
   const int last3_is_last =
       map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[0]];
+  const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[0]];
+  const int bwd_is_last = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idxes[0]];
+  const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[0]];
+
   const int last3_is_last2 =
       map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[1]];
+  const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]];
+  const int bwd_is_last2 = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idxes[1]];
+
   const int gld_is_last3 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[2]];
-  const int last3_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
+  const int bwd_is_last3 = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idxes[2]];
 
-  const int last4_is_last =
-      map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[0]];
-  const int last4_is_last2 =
-      map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[1]];
-  const int last4_is_last3 =
-      map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[2]];
+  const int bwd_is_gld = map[cpi->bwd_fb_idx] == map[cpi->gld_fb_idx];
 
-  const int last4_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
-  const int gld_is_last4 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[3]];
-#else  // CONFIG_EXT_REFS
+  const int last2_is_alt = map[cpi->lst_fb_idxes[1]] == map[cpi->alt_fb_idx];
+  const int last3_is_alt = map[cpi->lst_fb_idxes[2]] == map[cpi->alt_fb_idx];
+  const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
+  const int bwd_is_alt = map[cpi->bwd_fb_idx] == map[cpi->alt_fb_idx];
+#else
   const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idx];
-#if CONFIG_BIDIR_PRED
-  const int bwd_is_last = map[cpi->bwd_fb_idx] == map[cpi->lst_fb_idx];
-#endif  // CONFIG_BIDIR_PRED
+  const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
   const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idx];
 #endif  // CONFIG_EXT_REFS
-  const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
 
   int flags = VP9_REFFRAME_ALL;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // Disable the use of BWDREF_FRAME for non-bipredictive frames.
   if (!(cpi->rc.is_bipred_frame || cpi->rc.is_last_bipred_frame))
     flags &= ~VP9_BWD_FLAG;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
-  if (gld_is_alt || gld_is_last)
+  if (gld_is_last || gld_is_alt)
     flags &= ~VP9_GOLD_FLAG;
 
   if (cpi->rc.frames_till_gf_update_due == INT_MAX)
@@ -4641,22 +4602,18 @@
     flags &= ~VP9_ALT_FLAG;
 
 #if CONFIG_EXT_REFS
-  if (last4_is_alt || last4_is_last || last4_is_last2 || last4_is_last3)
-    flags &= ~VP9_LAST4_FLAG;
-
-  if (last3_is_alt || last3_is_last || last3_is_last2)
-    flags &= ~VP9_LAST3_FLAG;
-
-  if (last2_is_alt || last2_is_last)
+  if (last2_is_last || last2_is_alt)
     flags &= ~VP9_LAST2_FLAG;
 
-  if (gld_is_last4 || gld_is_last3 || gld_is_last2)
+  if (last3_is_last || last3_is_last2 || last3_is_alt)
+    flags &= ~VP9_LAST3_FLAG;
+
+  if (gld_is_last2 || gld_is_last3)
     flags &= ~VP9_GOLD_FLAG;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  if (bwd_is_last && (flags & VP9_BWD_FLAG))
+
+  if ((bwd_is_last || bwd_is_last2 || bwd_is_last3 ||
+       bwd_is_gld || bwd_is_alt) && (flags & VP9_BWD_FLAG))
     flags &= ~VP9_BWD_FLAG;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 
   return flags;
@@ -4724,9 +4681,9 @@
       (cpi->rc.source_alt_ref_active && !cpi->refresh_alt_ref_frame);
   }
   cm->ref_frame_sign_bias[ALTREF_FRAME] = arf_sign_bias;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   cm->ref_frame_sign_bias[BWDREF_FRAME] = cm->ref_frame_sign_bias[ALTREF_FRAME];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 }
 
 static int setup_interp_filter_search_mask(VP10_COMP *cpi) {
@@ -4751,18 +4708,15 @@
         (ref_total[LAST3_FRAME] == 0 ||
          cpi->interp_filter_selected[LAST3_FRAME][ifilter] * 50
          < ref_total[LAST3_FRAME]) &&
-        (ref_total[LAST4_FRAME] == 0 ||
-         cpi->interp_filter_selected[LAST4_FRAME][ifilter] * 50
-         < ref_total[LAST4_FRAME]) &&
 #endif  // CONFIG_EXT_REFS
         (ref_total[GOLDEN_FRAME] == 0 ||
          cpi->interp_filter_selected[GOLDEN_FRAME][ifilter] * 50
            < ref_total[GOLDEN_FRAME]) &&
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
         (ref_total[BWDREF_FRAME] == 0 ||
          cpi->interp_filter_selected[BWDREF_FRAME][ifilter] * 50
            < ref_total[BWDREF_FRAME]) &&
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
         (ref_total[ALTREF_FRAME] == 0 ||
          cpi->interp_filter_selected[ALTREF_FRAME][ifilter] * 50
            < ref_total[ALTREF_FRAME]))
@@ -4841,7 +4795,7 @@
   // Set the arf sign bias for this frame.
   set_arf_sign_bias(cpi);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // NOTE:
   // (1) Move the setup of the ref_frame_flags upfront as it would be
   //     determined by the current frame properties;
@@ -4881,6 +4835,7 @@
     // Update the LAST_FRAME in the reference frame buffer.
     vp10_update_reference_frames(cpi);
 
+    // Update frame flags
     cpi->frame_flags &= ~FRAMEFLAGS_GOLDEN;
     cpi->frame_flags &= ~FRAMEFLAGS_BWDREF;
     cpi->frame_flags &= ~FRAMEFLAGS_ALTREF;
@@ -4897,7 +4852,7 @@
 
     return;
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Set default state for segment based loop filter update flags.
   cm->lf.mode_ref_delta_update = 0;
@@ -5000,10 +4955,10 @@
   cm->frame_to_show->render_width  = cm->render_width;
   cm->frame_to_show->render_height = cm->render_height;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // TODO(zoeliu): For non-ref frames, loop filtering may need to be turned
   // off.
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Pick the loop filter level for the frame.
   loopfilter_frame(cpi, cm);
@@ -5017,13 +4972,13 @@
     dump_filtered_recon_frames(cpi);
 #endif  // DUMP_RECON_FRAMES
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (cpi->rc.is_last_bipred_frame) {
     // NOTE: If the current frame is a LAST_BIPRED_FRAME, next it is needed
     //       to show the BWDREF_FRAME.
     cpi->existing_fb_idx_to_show = cpi->bwd_fb_idx;
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   if (cm->seg.update_map)
     update_reference_segmentation_map(cpi);
@@ -5063,16 +5018,16 @@
   else
     cpi->frame_flags &= ~FRAMEFLAGS_ALTREF;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (cpi->refresh_bwd_ref_frame == 1)
     cpi->frame_flags |= FRAMEFLAGS_BWDREF;
   else
     cpi->frame_flags &= ~FRAMEFLAGS_BWDREF;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
-#if CONFIG_EXT_REFS || !CONFIG_BIDIR_PRED
+#if !CONFIG_EXT_REFS
   cpi->ref_frame_flags = get_ref_frame_flags(cpi);
-#endif  // CONFIG_EXT_REFS || !CONFIG_BIDIR_PRED
+#endif  // !CONFIG_EXT_REFS
 
 #if CONFIG_EXT_REFS
   cm->last3_frame_type = cm->last2_frame_type;
@@ -5108,20 +5063,20 @@
     cm->last_show_frame = cm->show_frame;
 
   if (cm->show_frame) {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     // TODO(zoeliu): We may only swamp mi and prev_mi for those frames that are
     // being used as reference.
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     vp10_swap_mi_and_prev_mi(cm);
     // Don't increment frame counters if this was an altref buffer
     // update not a real frame
     ++cm->current_video_frame;
   }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // NOTE: Shall not refer to any frame not used as reference.
   if (cm->is_reference_frame)
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     cm->prev_frame = cm->cur_frame;
 }
 
@@ -5140,9 +5095,10 @@
   cpi->allow_encode_breakout = ENCODE_BREAKOUT_ENABLED;
   encode_frame_to_data_rate(cpi, size, dest, frame_flags);
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
+  // Donot do the post-encoding update for show_existing_frame==1.
   if (!cpi->common.show_existing_frame)
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     vp10_twopass_postencode_update(cpi);
 }
 
@@ -5259,9 +5215,9 @@
   return cm->frame_type == KEY_FRAME ||
          cpi->refresh_last_frame ||
          cpi->refresh_golden_frame ||
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
          cpi->refresh_bwd_ref_frame ||
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
          cpi->refresh_alt_ref_frame ||
          !cm->error_resilient_mode ||
          cm->lf.mode_ref_delta_update ||
@@ -5326,7 +5282,7 @@
   return arf_src_index;
 }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 static int get_brf_src_index(VP10_COMP *cpi) {
   int brf_src_index = 0;
   const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
@@ -5345,7 +5301,7 @@
 
   return brf_src_index;
 }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 static void check_src_altref(VP10_COMP *cpi,
                              const struct lookahead_entry *source) {
@@ -5496,9 +5452,9 @@
   struct lookahead_entry *last_source = NULL;
   struct lookahead_entry *source = NULL;
   int arf_src_index;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   int brf_src_index;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   int i;
 
   vpx_usec_timer_start(&cmptimer);
@@ -5520,12 +5476,12 @@
 
   cpi->refresh_last_frame = 1;
   cpi->refresh_golden_frame = 0;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   cpi->refresh_bwd_ref_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   cpi->refresh_alt_ref_frame = 0;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (oxcf->pass == 2 && cm->show_existing_frame) {
     // Manage the source buffer and flush out the source frame that has been
     // coded already; Also get prepared for PSNR calculation if needed.
@@ -5573,7 +5529,7 @@
 
     return 0;
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Should we encode an arf frame.
   arf_src_index = get_arf_src_index(cpi);
@@ -5614,11 +5570,9 @@
     rc->source_alt_ref_pending = 0;
   }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   rc->is_bwd_ref_frame = 0;
   brf_src_index = get_brf_src_index(cpi);
-  // TODO(zoeliu): Need to handle when alt-ref is disabled; Currently bwd-ref
-  //               works only when alt-ref is on.
   if (brf_src_index) {
     assert(brf_src_index <= rc->frames_to_key);
     if ((source = vp10_lookahead_peek(cpi->lookahead, brf_src_index)) != NULL) {
@@ -5633,7 +5587,7 @@
       rc->is_bwd_ref_frame = 1;
     }
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   if (!source) {
     // Get last frame source.
@@ -5759,7 +5713,7 @@
 
   vpx_clear_system_state();
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (cpi->rc.is_last_bipred_frame) {
     // NOTE(zoeliu): If the current frame is a last bi-predictive frame, it is
     //               needed next to show the BWDREF_FRAME.
@@ -5768,7 +5722,7 @@
   } else {
     cm->show_existing_frame = 0;
   }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   return 0;
 }
@@ -5884,7 +5838,6 @@
 #if CONFIG_EXT_REFS
       ref ^= VP9_LAST2_FLAG;
       ref ^= VP9_LAST3_FLAG;
-      ref ^= VP9_LAST4_FLAG;
 #endif  // CONFIG_EXT_REFS
     }
 
@@ -5907,7 +5860,6 @@
 #if CONFIG_EXT_REFS
       upd ^= VP9_LAST2_FLAG;
       upd ^= VP9_LAST3_FLAG;
-      upd ^= VP9_LAST4_FLAG;
 #endif  // CONFIG_EXT_REFS
     }
 
diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h
index fc699e6..b1315ac 100644
--- a/vp10/encoder/encoder.h
+++ b/vp10/encoder/encoder.h
@@ -75,10 +75,6 @@
   FRAME_CONTEXT fc;
 } CODING_CONTEXT;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-#define BIDIR_PRED_PERIOD  2
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-
 typedef enum {
   // encode_breakout is disabled.
   ENCODE_BREAKOUT_DISABLED = 0,
@@ -114,12 +110,12 @@
 typedef enum {
   FRAMEFLAGS_KEY    = 1 << 0,
   FRAMEFLAGS_GOLDEN = 1 << 1,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   FRAMEFLAGS_BWDREF = 1 << 2,
   FRAMEFLAGS_ALTREF = 1 << 3,
 #else
   FRAMEFLAGS_ALTREF = 1 << 2,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 } FRAMETYPE_FLAGS;
 
 typedef enum {
@@ -205,9 +201,9 @@
   // ----------------------------------------------------------------
 
   int enable_auto_arf;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   int enable_auto_brf;  // (b)ackward (r)ef (f)rame
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   int encode_breakout;  // early breakout : for video conf recommend 800
 
@@ -364,22 +360,22 @@
   int scaled_ref_idx[MAX_REF_FRAMES];
 #if CONFIG_EXT_REFS
   int lst_fb_idxes[LAST_REF_FRAMES];
-#else  // CONFIG_EXT_REFS
+#else
   int lst_fb_idx;
 #endif  // CONFIG_EXT_REFS
   int gld_fb_idx;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   int bwd_fb_idx;  // BWD_REF_FRAME
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   int alt_fb_idx;
 
   int last_show_frame_buf_idx;  // last show frame buffer index
 
   int refresh_last_frame;
   int refresh_golden_frame;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   int refresh_bwd_ref_frame;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   int refresh_alt_ref_frame;
 
   int ext_refresh_frame_flags_pending;
@@ -615,10 +611,10 @@
 #if CONFIG_ANS
   struct BufAnsCoder buf_ans;
 #endif
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   int refresh_frame_mask;
   int existing_fb_idx_to_show;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 } VP10_COMP;
 
 void vp10_initialize_enc(void);
@@ -680,7 +676,7 @@
 static INLINE int get_ref_frame_map_idx(const VP10_COMP *cpi,
                                         MV_REFERENCE_FRAME ref_frame) {
 #if CONFIG_EXT_REFS
-  if (ref_frame >= LAST_FRAME && ref_frame <= LAST4_FRAME)
+  if (ref_frame >= LAST_FRAME && ref_frame <= LAST3_FRAME)
     return cpi->lst_fb_idxes[ref_frame - 1];
 #else
   if (ref_frame == LAST_FRAME)
@@ -688,10 +684,10 @@
 #endif  // CONFIG_EXT_REFS
   else if (ref_frame == GOLDEN_FRAME)
     return cpi->gld_fb_idx;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   else if (ref_frame == BWDREF_FRAME)
     return cpi->bwd_fb_idx;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   else
     return cpi->alt_fb_idx;
 }
@@ -719,7 +715,7 @@
   return &cpi->upsampled_ref_bufs[buf_idx].buf;
 }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 static INLINE int enc_is_ref_frame_buf(VP10_COMP *cpi,
                                        RefCntBuffer *frame_buf) {
   MV_REFERENCE_FRAME ref_frame;
@@ -731,7 +727,7 @@
   }
   return (ref_frame <= ALTREF_FRAME);
 }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 static INLINE unsigned int get_token_alloc(int mb_rows, int mb_cols) {
   // TODO(JBB): double check we can't exceed this token count if we have a
@@ -774,15 +770,15 @@
          cpi->oxcf.enable_auto_arf;
 }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+// TODO(zoeliu): To set up cpi->oxcf.enable_auto_brf
+#if 0 && CONFIG_EXT_REFS
 static INLINE int is_bwdref_enabled(const VP10_COMP *const cpi) {
-  // NOTE(zoeliu): The enabling of backward prediction depends on the alt_ref
-  // period, and will be off when the alt_ref period is not sufficiently large.
-  return cpi->oxcf.mode != REALTIME && cpi->oxcf.lag_in_frames > 0;
-         // (zoeliu):
-         // && cpi->oxcf.enable_auto_brf && cpi->rc.bidir_pred_enabled;
+  // NOTE(zoeliu): The enabling of bi-predictive frames depends on the use of
+  //               alt_ref, and now will be off when the alt_ref interval is
+  //               not sufficiently large.
+  return is_altref_enabled(cpi) && cpi->oxcf.enable_auto_brf;
 }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 static INLINE void set_ref_ptrs(VP10_COMMON *cm, MACROBLOCKD *xd,
                                 MV_REFERENCE_FRAME ref0,
diff --git a/vp10/encoder/firstpass.c b/vp10/encoder/firstpass.c
index b37d39f..71a6662 100644
--- a/vp10/encoder/firstpass.c
+++ b/vp10/encoder/firstpass.c
@@ -1632,7 +1632,7 @@
   int mid_frame_idx;
   unsigned char arf_buffer_indices[MAX_ACTIVE_ARFS];
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // The use of bi-predictive frames are only enabled when following 3
   // conditions are met:
   // (1) Alt-ref is enabled;
@@ -1645,7 +1645,7 @@
           (rc->baseline_gf_interval - rc->source_alt_ref_pending);
   int bipred_group_end = 0;
   int bipred_frame_index = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   key_frame = cpi->common.frame_type == KEY_FRAME;
 
@@ -1671,10 +1671,10 @@
       return;
   }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   gf_group->bidir_pred_enabled[frame_index] = 0;
   gf_group->brf_src_offset[frame_index] = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Deduct the boost bits for arf (or gf if it is not a key frame)
   // from the group total.
@@ -1683,9 +1683,9 @@
 
   frame_index++;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   bipred_frame_index++;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Store the bits to spend on the ARF if there is one.
   if (rc->source_alt_ref_pending) {
@@ -1701,11 +1701,11 @@
       arf_buffer_indices[cpi->multi_arf_last_grp_enabled &&
                          rc->source_alt_ref_active];
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     gf_group->bidir_pred_enabled[frame_index] = 0;
     gf_group->brf_src_offset[frame_index] = 0;
     // NOTE: "bidir_pred_frame_index" stays unchanged for ARF_UPDATE frames.
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
     ++frame_index;
 
@@ -1752,7 +1752,7 @@
     target_frame_size = clamp(target_frame_size, 0,
                               VPXMIN(max_bits, (int)total_group_bits));
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     // NOTE: BIDIR_PRED is only enabled when the length of the bi-predictive
     //       frame group interval is strictly smaller than that of the GOLDEN
     //       FRAME group interval.
@@ -1787,15 +1787,15 @@
           bipred_group_end = 1;
       }
     } else {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       gf_group->update_type[frame_index] = LF_UPDATE;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       gf_group->bidir_pred_enabled[frame_index] = 0;
       gf_group->brf_src_offset[frame_index] = 0;
     }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     if (gf_group->update_type[frame_index] == BRF_UPDATE) {
       // Boost up the allocated bits on BWDREF_FRAME
       gf_group->rf_level[frame_index] = INTER_HIGH;
@@ -1812,12 +1812,12 @@
       gf_group->rf_level[frame_index] = INTER_NORMAL;
       gf_group->bit_allocation[frame_index] = target_frame_size;
     } else {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       gf_group->rf_level[frame_index] = INTER_NORMAL;
       gf_group->bit_allocation[frame_index] = target_frame_size;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
     ++frame_index;
   }
@@ -1844,10 +1844,10 @@
     gf_group->update_type[frame_index] = GF_UPDATE;
     gf_group->rf_level[frame_index] = GF_ARF_STD;
   }
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   gf_group->bidir_pred_enabled[frame_index] = 0;
   gf_group->brf_src_offset[frame_index] = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Note whether multi-arf was enabled this group for next time.
   cpi->multi_arf_last_grp_enabled = cpi->multi_arf_enabled;
@@ -2060,12 +2060,12 @@
 
   rc->frames_till_gf_update_due = rc->baseline_gf_interval;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   rc->bipred_group_interval = BFG_INTERVAL;
   // The minimum bi-predictive frame group interval is 2.
   if (rc->bipred_group_interval < 2)
     rc->bipred_group_interval = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   // Reset the file position.
   reset_fpf_position(twopass, start_pos);
@@ -2508,46 +2508,46 @@
   TWO_PASS *const twopass = &cpi->twopass;
 
   cpi->rc.is_src_frame_alt_ref = 0;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   cpi->rc.is_bwd_ref_frame = 0;
   cpi->rc.is_last_bipred_frame = 0;
   cpi->rc.is_bipred_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   switch (twopass->gf_group.update_type[twopass->gf_group.index]) {
     case KF_UPDATE:
       cpi->refresh_last_frame = 1;
       cpi->refresh_golden_frame = 1;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 1;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 1;
       break;
 
     case LF_UPDATE:
       cpi->refresh_last_frame = 1;
       cpi->refresh_golden_frame = 0;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
 
     case GF_UPDATE:
       cpi->refresh_last_frame = 1;
       cpi->refresh_golden_frame = 1;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       break;
 
     case OVERLAY_UPDATE:
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 1;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 0;
       cpi->rc.is_src_frame_alt_ref = 1;
       break;
@@ -2555,13 +2555,13 @@
     case ARF_UPDATE:
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 0;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       cpi->refresh_alt_ref_frame = 1;
       break;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     case BRF_UPDATE:
       cpi->refresh_last_frame = 0;
       cpi->refresh_golden_frame = 0;
@@ -2585,7 +2585,7 @@
       cpi->refresh_alt_ref_frame = 0;
       cpi->rc.is_bipred_frame = 1;
       break;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
     default:
       assert(0);
diff --git a/vp10/encoder/firstpass.h b/vp10/encoder/firstpass.h
index b65c995..c9f4ad3 100644
--- a/vp10/encoder/firstpass.h
+++ b/vp10/encoder/firstpass.h
@@ -39,12 +39,12 @@
 } FIRSTPASS_MB_STATS;
 #endif
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 // Length of the bi-predictive frame group (BFG)
 // NOTE: Currently each BFG contains one backward ref (BWF) frame plus a certain
 //       number of bi-predictive frames.
 #define BFG_INTERVAL          2
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 #define VLOW_MOTION_THRESHOLD 950
 
@@ -79,14 +79,14 @@
   GF_UPDATE = 2,
   ARF_UPDATE = 3,
   OVERLAY_UPDATE = 4,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   BRF_UPDATE = 5,  // Backward Reference Frame
   LAST_BIPRED_UPDATE = 6,  // Last Bi-predictive Frame
   BIPRED_UPDATE = 7,  // Bi-predictive Frame, but not the last one
   FRAME_UPDATE_TYPES = 8
 #else
   FRAME_UPDATE_TYPES = 5
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 } FRAME_UPDATE_TYPE;
 
 #define FC_ANIMATION_THRESH 0.15
@@ -103,10 +103,10 @@
   unsigned char arf_src_offset[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char arf_update_idx[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char arf_ref_idx[(MAX_LAG_BUFFERS * 2) + 1];
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   unsigned char brf_src_offset[(MAX_LAG_BUFFERS * 2) + 1];
   unsigned char bidir_pred_enabled[(MAX_LAG_BUFFERS * 2) + 1];
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   int bit_allocation[(MAX_LAG_BUFFERS * 2) + 1];
 } GF_GROUP;
 
diff --git a/vp10/encoder/ratectrl.c b/vp10/encoder/ratectrl.c
index f2a14c1..5dd42d4 100644
--- a/vp10/encoder/ratectrl.c
+++ b/vp10/encoder/ratectrl.c
@@ -240,11 +240,13 @@
   RATE_CONTROL *const rc = &cpi->rc;
 
   // Non-viewable frames are a special case and are treated as pure overhead.
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
+  // TODO(zoeliu): To further explore whether we should treat BWDREF_FRAME
+  //               differently, since it is a no-show frame.
   if (!cm->show_frame && !rc->is_bwd_ref_frame)
 #else
   if (!cm->show_frame)
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     rc->bits_off_target -= encoded_frame_size;
   else
     rc->bits_off_target += rc->avg_frame_bandwidth - encoded_frame_size;
@@ -949,25 +951,24 @@
 
 int vp10_frame_type_qdelta(const VP10_COMP *cpi, int rf_level, int q) {
   static const double rate_factor_deltas[RATE_FACTOR_LEVELS] = {
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
     1.00,  // INTER_NORMAL
+#if CONFIG_EXT_REFS
     0.80,  // INTER_LOW
     1.25,  // INTER_HIGH
 #else
-    1.00,  // INTER_NORMAL
     1.00,  // INTER_HIGH
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     1.50,  // GF_ARF_LOW
     1.75,  // GF_ARF_STD
     2.00,  // KF_STD
   };
   static const FRAME_TYPE frame_type[RATE_FACTOR_LEVELS] =
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       { INTER_FRAME, INTER_FRAME, INTER_FRAME,
         INTER_FRAME, INTER_FRAME, KEY_FRAME };
 #else
       {INTER_FRAME, INTER_FRAME, INTER_FRAME, INTER_FRAME, KEY_FRAME};
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   const VP10_COMMON *const cm = &cpi->common;
   int qdelta = vp10_compute_qdelta_by_rate(&cpi->rc, frame_type[rf_level],
                                           q, rate_factor_deltas[rf_level],
@@ -1328,12 +1329,12 @@
 
   // Actual bits spent
   rc->total_actual_bits += rc->projected_frame_size;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   rc->total_target_bits += (cm->show_frame || rc->is_bwd_ref_frame) ?
                             rc->avg_frame_bandwidth : 0;
 #else
   rc->total_target_bits += cm->show_frame ? rc->avg_frame_bandwidth : 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   rc->total_target_vs_actual = rc->total_actual_bits - rc->total_target_bits;
 
@@ -1348,11 +1349,11 @@
   if (cm->frame_type == KEY_FRAME)
     rc->frames_since_key = 0;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   if (cm->show_frame || rc->is_bwd_ref_frame) {
 #else
   if (cm->show_frame) {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     rc->frames_since_key++;
     rc->frames_to_key--;
   }
diff --git a/vp10/encoder/ratectrl.h b/vp10/encoder/ratectrl.h
index e1d597c..f70429b 100644
--- a/vp10/encoder/ratectrl.h
+++ b/vp10/encoder/ratectrl.h
@@ -28,7 +28,7 @@
 #define MAX_GF_INTERVAL     16
 #define FIXED_GF_INTERVAL   8    // Used in some testing modes only
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 typedef enum {
   INTER_NORMAL = 0,
   INTER_LOW = 1,
@@ -47,7 +47,7 @@
   KF_STD = 4,
   RATE_FACTOR_LEVELS = 5
 } RATE_FACTOR_LEVEL;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 // Internal frame scaling level.
 typedef enum {
@@ -102,7 +102,7 @@
   int source_alt_ref_active;
   int is_src_frame_alt_ref;
 
-#if CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // Length of the bi-predictive frame group interval
   int bipred_group_interval;
 
@@ -111,7 +111,7 @@
   int is_bwd_ref_frame;
   int is_last_bipred_frame;
   int is_bipred_frame;
-#endif  // CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   int avg_frame_bandwidth;  // Average frame size target for clip
   int min_frame_bandwidth;  // Minimum allocation used for any frame
diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index fb1482e..3956076 100644
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -224,10 +224,10 @@
 };
 static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
   128, 144, 128, 128, 144,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   // TODO(zoeliu): To adjust further following factor values.
   128, 128, 128
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 };
 
 int vp10_compute_rd_mult(const VP10_COMP *cpi, int qindex) {
@@ -769,27 +769,19 @@
 #if CONFIG_EXT_REFS
     rd->thresh_mult[THR_NEARESTL2] = 300;
     rd->thresh_mult[THR_NEARESTL3] = 300;
-    rd->thresh_mult[THR_NEARESTL4] = 300;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
     rd->thresh_mult[THR_NEARESTB] = 300;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
-    rd->thresh_mult[THR_NEARESTG] = 300;
     rd->thresh_mult[THR_NEARESTA] = 300;
+    rd->thresh_mult[THR_NEARESTG] = 300;
   } else {
     rd->thresh_mult[THR_NEARESTMV] = 0;
 #if CONFIG_EXT_REFS
     rd->thresh_mult[THR_NEARESTL2] = 0;
     rd->thresh_mult[THR_NEARESTL3] = 0;
-    rd->thresh_mult[THR_NEARESTL4] = 0;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
     rd->thresh_mult[THR_NEARESTB] = 0;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
-    rd->thresh_mult[THR_NEARESTG] = 0;
     rd->thresh_mult[THR_NEARESTA] = 0;
+    rd->thresh_mult[THR_NEARESTG] = 0;
   }
 
   rd->thresh_mult[THR_DC] += 1000;
@@ -798,11 +790,7 @@
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEWL2] += 1000;
   rd->thresh_mult[THR_NEWL3] += 1000;
-  rd->thresh_mult[THR_NEWL4] += 1000;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   rd->thresh_mult[THR_NEWB] += 1000;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEWA] += 1000;
   rd->thresh_mult[THR_NEWG] += 1000;
@@ -811,11 +799,7 @@
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEARL2] += 1000;
   rd->thresh_mult[THR_NEARL3] += 1000;
-  rd->thresh_mult[THR_NEARL4] += 1000;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   rd->thresh_mult[THR_NEARB] += 1000;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEARA] += 1000;
   rd->thresh_mult[THR_NEARG] += 1000;
@@ -825,25 +809,17 @@
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_NEWFROMNEARL2] += 1000;
   rd->thresh_mult[THR_NEWFROMNEARL3] += 1000;
-  rd->thresh_mult[THR_NEWFROMNEARL4] += 1000;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   rd->thresh_mult[THR_NEWFROMNEARB] += 1000;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
-  rd->thresh_mult[THR_NEWFROMNEARG] += 1000;
   rd->thresh_mult[THR_NEWFROMNEARA] += 1000;
+  rd->thresh_mult[THR_NEWFROMNEARG] += 1000;
 #endif  // CONFIG_EXT_INTER
 
   rd->thresh_mult[THR_ZEROMV] += 2000;
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_ZEROL2] += 2000;
   rd->thresh_mult[THR_ZEROL3] += 2000;
-  rd->thresh_mult[THR_ZEROL4] += 2000;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   rd->thresh_mult[THR_ZEROB] += 2000;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_ZEROG] += 2000;
   rd->thresh_mult[THR_ZEROA] += 2000;
@@ -851,29 +827,50 @@
   rd->thresh_mult[THR_TM] += 1000;
 
 #if CONFIG_EXT_INTER
-  rd->thresh_mult[THR_COMP_NEAREST_NEARESTLA] += 1000;
-  rd->thresh_mult[THR_COMP_NEAREST_NEARESTGA] += 1000;
-  rd->thresh_mult[THR_COMP_NEAREST_NEARLA] += 1200;
-  rd->thresh_mult[THR_COMP_NEAREST_NEARGA] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARESTLA] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARESTGA] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARLA] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARGA] += 1200;
-  rd->thresh_mult[THR_COMP_NEAREST_NEWLA] += 1500;
-  rd->thresh_mult[THR_COMP_NEAREST_NEWGA] += 1500;
-  rd->thresh_mult[THR_COMP_NEW_NEARESTLA] += 1500;
-  rd->thresh_mult[THR_COMP_NEW_NEARESTGA] += 1500;
-  rd->thresh_mult[THR_COMP_NEAR_NEWLA] += 1700;
-  rd->thresh_mult[THR_COMP_NEAR_NEWGA] += 1700;
-  rd->thresh_mult[THR_COMP_NEW_NEARLA] += 1700;
-  rd->thresh_mult[THR_COMP_NEW_NEARGA] += 1700;
-  rd->thresh_mult[THR_COMP_NEW_NEWLA] += 2000;
-  rd->thresh_mult[THR_COMP_NEW_NEWGA] += 2000;
-  rd->thresh_mult[THR_COMP_ZERO_ZEROLA] += 2500;
-  rd->thresh_mult[THR_COMP_ZERO_ZEROGA] += 2500;
 
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTLA] += 1000;
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2A] += 1000;
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3A] += 1000;
+#endif  // CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTGA] += 1000;
+#if CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTLB] += 1000;
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTL2B] += 1000;
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3B] += 1000;
+  rd->thresh_mult[THR_COMP_NEAREST_NEARESTGB] += 1000;
+#endif  // CONFIG_EXT_REFS
+
+#else  // CONFIG_EXT_INTER
+
+  rd->thresh_mult[THR_COMP_NEARESTLA] += 1000;
+#if CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_NEARESTL2A] += 1000;
+  rd->thresh_mult[THR_COMP_NEARESTL3A] += 1000;
+#endif  // CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_NEARESTGA] += 1000;
+#if CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_NEARESTLB] += 1000;
+  rd->thresh_mult[THR_COMP_NEARESTL2B] += 1000;
+  rd->thresh_mult[THR_COMP_NEARESTL3B] += 1000;
+  rd->thresh_mult[THR_COMP_NEARESTGB] += 1000;
+#endif  // CONFIG_EXT_REFS
+
+#endif  // CONFIG_EXT_INTER
+
+#if CONFIG_EXT_INTER
+
+  rd->thresh_mult[THR_COMP_NEAREST_NEARLA] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARESTLA] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARLA] += 1200;
+  rd->thresh_mult[THR_COMP_NEAREST_NEWLA] += 1500;
+  rd->thresh_mult[THR_COMP_NEW_NEARESTLA] += 1500;
+  rd->thresh_mult[THR_COMP_NEAR_NEWLA] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEARLA] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEWLA] += 2000;
+  rd->thresh_mult[THR_COMP_ZERO_ZEROLA] += 2500;
+
+#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAREST_NEARL2A] += 1200;
   rd->thresh_mult[THR_COMP_NEAR_NEARESTL2A] += 1200;
   rd->thresh_mult[THR_COMP_NEAR_NEARL2A] += 1200;
@@ -884,7 +881,6 @@
   rd->thresh_mult[THR_COMP_NEW_NEWL2A] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROL2A] += 2500;
 
-  rd->thresh_mult[THR_COMP_NEAREST_NEARESTL3A] += 1000;
   rd->thresh_mult[THR_COMP_NEAREST_NEARL3A] += 1200;
   rd->thresh_mult[THR_COMP_NEAR_NEARESTL3A] += 1200;
   rd->thresh_mult[THR_COMP_NEAR_NEARL3A] += 1200;
@@ -894,62 +890,62 @@
   rd->thresh_mult[THR_COMP_NEW_NEARL3A] += 1700;
   rd->thresh_mult[THR_COMP_NEW_NEWL3A] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROL3A] += 2500;
+#endif  // CONFIG_EXT_REFS
 
-  rd->thresh_mult[THR_COMP_NEAREST_NEARESTL4A] += 1000;
-  rd->thresh_mult[THR_COMP_NEAREST_NEARL4A] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARESTL4A] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARL4A] += 1200;
-  rd->thresh_mult[THR_COMP_NEAREST_NEWL4A] += 1500;
-  rd->thresh_mult[THR_COMP_NEW_NEARESTL4A] += 1500;
-  rd->thresh_mult[THR_COMP_NEAR_NEWL4A] += 1700;
-  rd->thresh_mult[THR_COMP_NEW_NEARL4A] += 1700;
-  rd->thresh_mult[THR_COMP_NEW_NEWL4A] += 2000;
-  rd->thresh_mult[THR_COMP_ZERO_ZEROL4A] += 2500;
+  rd->thresh_mult[THR_COMP_NEAREST_NEARGA] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARESTGA] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARGA] += 1200;
+  rd->thresh_mult[THR_COMP_NEAREST_NEWGA] += 1500;
+  rd->thresh_mult[THR_COMP_NEW_NEARESTGA] += 1500;
+  rd->thresh_mult[THR_COMP_NEAR_NEWGA] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEARGA] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEWGA] += 2000;
+  rd->thresh_mult[THR_COMP_ZERO_ZEROGA] += 2500;
 
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
-  rd->thresh_mult[THR_COMP_NEAREST_NEARESTLB] += 1000;
-  rd->thresh_mult[THR_COMP_NEAREST_NEARESTGB] += 1000;
-
+#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEAREST_NEARLB] += 1200;
-  rd->thresh_mult[THR_COMP_NEAREST_NEARGB] += 1200;
   rd->thresh_mult[THR_COMP_NEAR_NEARESTLB] += 1200;
-  rd->thresh_mult[THR_COMP_NEAR_NEARESTGB] += 1200;
-
+  rd->thresh_mult[THR_COMP_NEAR_NEARLB] += 1200;
   rd->thresh_mult[THR_COMP_NEAREST_NEWLB] += 1500;
-  rd->thresh_mult[THR_COMP_NEAREST_NEWGB] += 1500;
   rd->thresh_mult[THR_COMP_NEW_NEARESTLB] += 1500;
-  rd->thresh_mult[THR_COMP_NEW_NEARESTGB] += 1500;
-
   rd->thresh_mult[THR_COMP_NEAR_NEWLB] += 1700;
-  rd->thresh_mult[THR_COMP_NEAR_NEWGB] += 1700;
   rd->thresh_mult[THR_COMP_NEW_NEARLB] += 1700;
-  rd->thresh_mult[THR_COMP_NEW_NEARGB] += 1700;
-
   rd->thresh_mult[THR_COMP_NEW_NEWLB] += 2000;
-  rd->thresh_mult[THR_COMP_NEW_NEWGB] += 2000;
-
   rd->thresh_mult[THR_COMP_ZERO_ZEROLB] += 2500;
+
+  rd->thresh_mult[THR_COMP_NEAREST_NEARL2B] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARESTL2B] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARL2B] += 1200;
+  rd->thresh_mult[THR_COMP_NEAREST_NEWL2B] += 1500;
+  rd->thresh_mult[THR_COMP_NEW_NEARESTL2B] += 1500;
+  rd->thresh_mult[THR_COMP_NEAR_NEWL2B] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEARL2B] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEWL2B] += 2000;
+  rd->thresh_mult[THR_COMP_ZERO_ZEROL2B] += 2500;
+
+  rd->thresh_mult[THR_COMP_NEAREST_NEARL3B] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARESTL3B] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARL3B] += 1200;
+  rd->thresh_mult[THR_COMP_NEAREST_NEWL3B] += 1500;
+  rd->thresh_mult[THR_COMP_NEW_NEARESTL3B] += 1500;
+  rd->thresh_mult[THR_COMP_NEAR_NEWL3B] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEARL3B] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEWL3B] += 2000;
+  rd->thresh_mult[THR_COMP_ZERO_ZEROL3B] += 2500;
+
+  rd->thresh_mult[THR_COMP_NEAREST_NEARGB] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARESTGB] += 1200;
+  rd->thresh_mult[THR_COMP_NEAR_NEARGB] += 1200;
+  rd->thresh_mult[THR_COMP_NEAREST_NEWGB] += 1500;
+  rd->thresh_mult[THR_COMP_NEW_NEARESTGB] += 1500;
+  rd->thresh_mult[THR_COMP_NEAR_NEWGB] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEARGB] += 1700;
+  rd->thresh_mult[THR_COMP_NEW_NEWGB] += 2000;
   rd->thresh_mult[THR_COMP_ZERO_ZEROGB] += 2500;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 
 #else  // CONFIG_EXT_INTER
 
-  rd->thresh_mult[THR_COMP_NEARESTLA] += 1000;
-#if CONFIG_EXT_REFS
-  rd->thresh_mult[THR_COMP_NEARESTL2A] += 1000;
-  rd->thresh_mult[THR_COMP_NEARESTL3A] += 1000;
-  rd->thresh_mult[THR_COMP_NEARESTL4A] += 1000;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  rd->thresh_mult[THR_COMP_NEARESTLB] += 1000;
-  rd->thresh_mult[THR_COMP_NEARESTGB] += 1000;
-#endif  // CONFIG_BIDIR_PRED
-#endif  // CONFIG_EXT_REFS
-  rd->thresh_mult[THR_COMP_NEARESTGA] += 1000;
-
   rd->thresh_mult[THR_COMP_NEARLA] += 1500;
   rd->thresh_mult[THR_COMP_NEWLA] += 2000;
 #if CONFIG_EXT_REFS
@@ -957,78 +953,79 @@
   rd->thresh_mult[THR_COMP_NEWL2A] += 2000;
   rd->thresh_mult[THR_COMP_NEARL3A] += 1500;
   rd->thresh_mult[THR_COMP_NEWL3A] += 2000;
-  rd->thresh_mult[THR_COMP_NEARL4A] += 1500;
-  rd->thresh_mult[THR_COMP_NEWL4A] += 2000;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  rd->thresh_mult[THR_COMP_NEARLB] += 1500;
-  rd->thresh_mult[THR_COMP_NEARGB] += 1500;
-  rd->thresh_mult[THR_COMP_NEWLB] += 2000;
-  rd->thresh_mult[THR_COMP_NEWGB] += 2000;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_NEARGA] += 1500;
   rd->thresh_mult[THR_COMP_NEWGA] += 2000;
 
+#if CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_NEARLB] += 1500;
+  rd->thresh_mult[THR_COMP_NEWLB] += 2000;
+  rd->thresh_mult[THR_COMP_NEARL2B] += 1500;
+  rd->thresh_mult[THR_COMP_NEWL2B] += 2000;
+  rd->thresh_mult[THR_COMP_NEARL3B] += 1500;
+  rd->thresh_mult[THR_COMP_NEWL3B] += 2000;
+  rd->thresh_mult[THR_COMP_NEARGB] += 1500;
+  rd->thresh_mult[THR_COMP_NEWGB] += 2000;
+#endif  // CONFIG_EXT_REFS
+
   rd->thresh_mult[THR_COMP_ZEROLA] += 2500;
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_ZEROL2A] += 2500;
   rd->thresh_mult[THR_COMP_ZEROL3A] += 2500;
-  rd->thresh_mult[THR_COMP_ZEROL4A] += 2500;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  rd->thresh_mult[THR_COMP_ZEROLB] += 2500;
-  rd->thresh_mult[THR_COMP_ZEROGB] += 2500;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_ZEROGA] += 2500;
 
+#if CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_ZEROLB] += 2500;
+  rd->thresh_mult[THR_COMP_ZEROL2B] += 2500;
+  rd->thresh_mult[THR_COMP_ZEROL3B] += 2500;
+  rd->thresh_mult[THR_COMP_ZEROGB] += 2500;
+#endif  // CONFIG_EXT_REFS
+
 #endif  // CONFIG_EXT_INTER
 
   rd->thresh_mult[THR_H_PRED] += 2000;
   rd->thresh_mult[THR_V_PRED] += 2000;
-  rd->thresh_mult[THR_D45_PRED ] += 2500;
   rd->thresh_mult[THR_D135_PRED] += 2500;
-  rd->thresh_mult[THR_D117_PRED] += 2500;
-  rd->thresh_mult[THR_D153_PRED] += 2500;
   rd->thresh_mult[THR_D207_PRED] += 2500;
+  rd->thresh_mult[THR_D153_PRED] += 2500;
   rd->thresh_mult[THR_D63_PRED] += 2500;
+  rd->thresh_mult[THR_D117_PRED] += 2500;
+  rd->thresh_mult[THR_D45_PRED ] += 2500;
 
 #if CONFIG_EXT_INTER
   rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL   ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARL   ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEWL    ] += 2000;
+
 #if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL2  ] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL3  ] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL4  ] += 1500;
-#endif  // CONFIG_EXT_REFS
-  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROG   ] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROA   ] += 1500;
-
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL] += 1500;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL2] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL3] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL4] += 1500;
-#endif  // CONFIG_EXT_REFS
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTG] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTA] += 1500;
-
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARL   ] += 1500;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_INTERINTRA_NEARL2  ] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARL3  ] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARL4  ] += 1500;
-#endif  // CONFIG_EXT_REFS
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARG   ] += 1500;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEARA   ] += 1500;
-
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEWL    ] += 2000;
-#if CONFIG_EXT_REFS
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWL2   ] += 2000;
+
+  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROL3  ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTL3] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARL3  ] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWL3   ] += 2000;
-  rd->thresh_mult[THR_COMP_INTERINTRA_NEWL4   ] += 2000;
 #endif  // CONFIG_EXT_REFS
+
+  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROG   ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTG] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARG   ] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWG    ] += 2000;
+
+#if CONFIG_EXT_REFS
+  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROB   ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTB] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARB   ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEWB    ] += 2000;
+#endif  // CONFIG_EXT_REFS
+
+  rd->thresh_mult[THR_COMP_INTERINTRA_ZEROA   ] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARESTA] += 1500;
+  rd->thresh_mult[THR_COMP_INTERINTRA_NEARA   ] += 1500;
   rd->thresh_mult[THR_COMP_INTERINTRA_NEWA    ] += 2000;
 #endif  // CONFIG_EXT_INTER
 }
@@ -1036,16 +1033,13 @@
 void vp10_set_rd_speed_thresholds_sub8x8(VP10_COMP *cpi) {
   static const int thresh_mult[2][MAX_REFS] = {
 #if CONFIG_EXT_REFS
-    {2500, 2500, 2500, 2500, 2500, 2500, 4500, 4500, 4500, 4500, 4500, 2500},
-    {2000, 2000, 2000, 2000, 2000, 2000, 4000, 4000, 4000, 4000, 4000, 2000}
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-    {2500, 2500, 2500, 2500, 4500, 4500, 4500, 4500, 2500},
-    {2000, 2000, 2000, 2000, 4000, 4000, 4000, 4000, 2000}
-#else  // CONFIG_BIDIR_PRED
-    {2500, 2500, 2500, 4500, 4500, 2500},
-    {2000, 2000, 2000, 4000, 4000, 2000}
-#endif  // CONFIG_BIDIR_PRED
+    { 2500, 2500, 2500, 2500, 2500, 2500, 4500, 4500,
+      4500, 4500, 4500, 4500, 4500, 4500, 2500 },
+    { 2000, 2000, 2000, 2000, 2000, 2000, 4000, 4000,
+      4000, 4000, 4000, 4000, 4000, 4000, 2000 }
+#else
+    { 2500, 2500, 2500, 4500, 4500, 2500 },
+    { 2000, 2000, 2000, 4000, 4000, 2000 }
 #endif  // CONFIG_EXT_REFS
   };
   RD_OPT *const rd = &cpi->rd;
diff --git a/vp10/encoder/rd.h b/vp10/encoder/rd.h
index 86872c0..3b5f470 100644
--- a/vp10/encoder/rd.h
+++ b/vp10/encoder/rd.h
@@ -46,41 +46,25 @@
 #if CONFIG_EXT_REFS
 
 #if CONFIG_EXT_INTER
-#define MAX_MODES 114
+#define MAX_MODES 144
 #else  // CONFIG_EXT_INTER
-#define MAX_MODES 54
+#define MAX_MODES 66
 #endif  // CONFIG_EXT_INTER
 
 #else  // CONFIG_EXT_REFS
 
-#if CONFIG_BIDIR_PRED
-
-#if CONFIG_EXT_INTER
-#define MAX_MODES 80
-#else  // CONFIG_EXT_INTER
-#define MAX_MODES 42
-#endif  // CONFIG_EXT_INTER
-
-#else  // CONFIG_BIDIR_PRED
-
 #if CONFIG_EXT_INTER
 #define MAX_MODES 57
 #else  // CONFIG_EXT_INTER
 #define MAX_MODES 30
 #endif  // CONFIG_EXT_INTER
 
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
 
 #if CONFIG_EXT_REFS
-#define MAX_REFS  12
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-#define MAX_REFS  9
-#else  // CONFIG_BIDIR_PRED
+#define MAX_REFS  15
+#else
 #define MAX_REFS  6
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 
 #define RD_THRESH_MAX_FACT 64
@@ -93,11 +77,7 @@
 #if CONFIG_EXT_REFS
   THR_NEARESTL2,
   THR_NEARESTL3,
-  THR_NEARESTL4,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   THR_NEARESTB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_NEARESTA,
   THR_NEARESTG,
@@ -108,11 +88,7 @@
 #if CONFIG_EXT_REFS
   THR_NEWL2,
   THR_NEWL3,
-  THR_NEWL4,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   THR_NEWB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_NEWA,
   THR_NEWG,
@@ -121,11 +97,7 @@
 #if CONFIG_EXT_REFS
   THR_NEARL2,
   THR_NEARL3,
-  THR_NEARL4,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   THR_NEARB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_NEARA,
   THR_NEARG,
@@ -135,11 +107,7 @@
 #if CONFIG_EXT_REFS
   THR_NEWFROMNEARL2,
   THR_NEWFROMNEARL3,
-  THR_NEWFROMNEARL4,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   THR_NEWFROMNEARB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_NEWFROMNEARA,
   THR_NEWFROMNEARG,
@@ -149,64 +117,56 @@
 #if CONFIG_EXT_REFS
   THR_ZEROL2,
   THR_ZEROL3,
-  THR_ZEROL4,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   THR_ZEROB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_ZEROG,
   THR_ZEROA,
 
 #if CONFIG_EXT_INTER
+
   THR_COMP_NEAREST_NEARESTLA,
 #if CONFIG_EXT_REFS
   THR_COMP_NEAREST_NEARESTL2A,
   THR_COMP_NEAREST_NEARESTL3A,
-  THR_COMP_NEAREST_NEARESTL4A,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  THR_COMP_NEAREST_NEARESTLB,
-  THR_COMP_NEAREST_NEARESTGB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_COMP_NEAREST_NEARESTGA,
+#if CONFIG_EXT_REFS
+  THR_COMP_NEAREST_NEARESTLB,
+  THR_COMP_NEAREST_NEARESTL2B,
+  THR_COMP_NEAREST_NEARESTL3B,
+  THR_COMP_NEAREST_NEARESTGB,
+#endif  // CONFIG_EXT_REFS
+
 #else  // CONFIG_EXT_INTER
+
   THR_COMP_NEARESTLA,
 #if CONFIG_EXT_REFS
   THR_COMP_NEARESTL2A,
   THR_COMP_NEARESTL3A,
-  THR_COMP_NEARESTL4A,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  THR_COMP_NEARESTLB,
-  THR_COMP_NEARESTGB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_COMP_NEARESTGA,
+#if CONFIG_EXT_REFS
+  THR_COMP_NEARESTLB,
+  THR_COMP_NEARESTL2B,
+  THR_COMP_NEARESTL3B,
+  THR_COMP_NEARESTGB,
+#endif  // CONFIG_EXT_REFS
+
 #endif  // CONFIG_EXT_INTER
 
   THR_TM,
 
 #if CONFIG_EXT_INTER
+
   THR_COMP_NEAR_NEARESTLA,
-  THR_COMP_NEAR_NEARESTGA,
   THR_COMP_NEAREST_NEARLA,
-  THR_COMP_NEAREST_NEARGA,
   THR_COMP_NEAR_NEARLA,
-  THR_COMP_NEAR_NEARGA,
   THR_COMP_NEW_NEARESTLA,
-  THR_COMP_NEW_NEARESTGA,
   THR_COMP_NEAREST_NEWLA,
-  THR_COMP_NEAREST_NEWGA,
   THR_COMP_NEW_NEARLA,
-  THR_COMP_NEW_NEARGA,
   THR_COMP_NEAR_NEWLA,
-  THR_COMP_NEAR_NEWGA,
   THR_COMP_NEW_NEWLA,
-  THR_COMP_NEW_NEWGA,
   THR_COMP_ZERO_ZEROLA,
-  THR_COMP_ZERO_ZEROGA,
 
 #if CONFIG_EXT_REFS
   THR_COMP_NEAR_NEARESTL2A,
@@ -228,37 +188,58 @@
   THR_COMP_NEAR_NEWL3A,
   THR_COMP_NEW_NEWL3A,
   THR_COMP_ZERO_ZEROL3A,
+#endif  // CONFIG_EXT_REFS
 
-  THR_COMP_NEAR_NEARESTL4A,
-  THR_COMP_NEAREST_NEARL4A,
-  THR_COMP_NEAR_NEARL4A,
-  THR_COMP_NEW_NEARESTL4A,
-  THR_COMP_NEAREST_NEWL4A,
-  THR_COMP_NEW_NEARL4A,
-  THR_COMP_NEAR_NEWL4A,
-  THR_COMP_NEW_NEWL4A,
-  THR_COMP_ZERO_ZEROL4A,
+  THR_COMP_NEAR_NEARESTGA,
+  THR_COMP_NEAREST_NEARGA,
+  THR_COMP_NEAR_NEARGA,
+  THR_COMP_NEW_NEARESTGA,
+  THR_COMP_NEAREST_NEWGA,
+  THR_COMP_NEW_NEARGA,
+  THR_COMP_NEAR_NEWGA,
+  THR_COMP_NEW_NEWGA,
+  THR_COMP_ZERO_ZEROGA,
 
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   THR_COMP_NEAR_NEARESTLB,
-  THR_COMP_NEAR_NEARESTGB,
   THR_COMP_NEAREST_NEARLB,
-  THR_COMP_NEAREST_NEARGB,
+  THR_COMP_NEAR_NEARLB,
   THR_COMP_NEW_NEARESTLB,
-  THR_COMP_NEW_NEARESTGB,
   THR_COMP_NEAREST_NEWLB,
-  THR_COMP_NEAREST_NEWGB,
   THR_COMP_NEW_NEARLB,
-  THR_COMP_NEW_NEARGB,
   THR_COMP_NEAR_NEWLB,
-  THR_COMP_NEAR_NEWGB,
   THR_COMP_NEW_NEWLB,
-  THR_COMP_NEW_NEWGB,
   THR_COMP_ZERO_ZEROLB,
+
+  THR_COMP_NEAR_NEARESTL2B,
+  THR_COMP_NEAREST_NEARL2B,
+  THR_COMP_NEAR_NEARL2B,
+  THR_COMP_NEW_NEARESTL2B,
+  THR_COMP_NEAREST_NEWL2B,
+  THR_COMP_NEW_NEARL2B,
+  THR_COMP_NEAR_NEWL2B,
+  THR_COMP_NEW_NEWL2B,
+  THR_COMP_ZERO_ZEROL2B,
+
+  THR_COMP_NEAR_NEARESTL3B,
+  THR_COMP_NEAREST_NEARL3B,
+  THR_COMP_NEAR_NEARL3B,
+  THR_COMP_NEW_NEARESTL3B,
+  THR_COMP_NEAREST_NEWL3B,
+  THR_COMP_NEW_NEARL3B,
+  THR_COMP_NEAR_NEWL3B,
+  THR_COMP_NEW_NEWL3B,
+  THR_COMP_ZERO_ZEROL3B,
+
+  THR_COMP_NEAR_NEARESTGB,
+  THR_COMP_NEAREST_NEARGB,
+  THR_COMP_NEAR_NEARGB,
+  THR_COMP_NEW_NEARESTGB,
+  THR_COMP_NEAREST_NEWGB,
+  THR_COMP_NEW_NEARGB,
+  THR_COMP_NEAR_NEWGB,
+  THR_COMP_NEW_NEWGB,
   THR_COMP_ZERO_ZEROGB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 
 #else  // CONFIG_EXT_INTER
@@ -270,31 +251,35 @@
   THR_COMP_NEWL2A,
   THR_COMP_NEARL3A,
   THR_COMP_NEWL3A,
-  THR_COMP_NEARL4A,
-  THR_COMP_NEWL4A,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  THR_COMP_NEARLB,
-  THR_COMP_NEWLB,
-  THR_COMP_NEARGB,
-  THR_COMP_NEWGB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_COMP_NEARGA,
   THR_COMP_NEWGA,
 
+#if CONFIG_EXT_REFS
+  THR_COMP_NEARLB,
+  THR_COMP_NEWLB,
+  THR_COMP_NEARL2B,
+  THR_COMP_NEWL2B,
+  THR_COMP_NEARL3B,
+  THR_COMP_NEWL3B,
+  THR_COMP_NEARGB,
+  THR_COMP_NEWGB,
+#endif  // CONFIG_EXT_REFS
+
   THR_COMP_ZEROLA,
 #if CONFIG_EXT_REFS
   THR_COMP_ZEROL2A,
   THR_COMP_ZEROL3A,
-  THR_COMP_ZEROL4A,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  THR_COMP_ZEROLB,
-  THR_COMP_ZEROGB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_COMP_ZEROGA,
+
+#if CONFIG_EXT_REFS
+  THR_COMP_ZEROLB,
+  THR_COMP_ZEROL2B,
+  THR_COMP_ZEROL3B,
+  THR_COMP_ZEROGB,
+#endif  // CONFIG_EXT_REFS
+
 #endif  // CONFIG_EXT_INTER
 
   THR_H_PRED,
@@ -322,11 +307,6 @@
   THR_COMP_INTERINTRA_NEARESTL3,
   THR_COMP_INTERINTRA_NEARL3,
   THR_COMP_INTERINTRA_NEWL3,
-
-  THR_COMP_INTERINTRA_ZEROL4,
-  THR_COMP_INTERINTRA_NEARESTL4,
-  THR_COMP_INTERINTRA_NEARL4,
-  THR_COMP_INTERINTRA_NEWL4,
 #endif  // CONFIG_EXT_REFS
 
   THR_COMP_INTERINTRA_ZEROG,
@@ -334,6 +314,13 @@
   THR_COMP_INTERINTRA_NEARG,
   THR_COMP_INTERINTRA_NEWG,
 
+#if CONFIG_EXT_REFS
+  THR_COMP_INTERINTRA_ZEROB,
+  THR_COMP_INTERINTRA_NEARESTB,
+  THR_COMP_INTERINTRA_NEARB,
+  THR_COMP_INTERINTRA_NEWB,
+#endif  // CONFIG_EXT_REFS
+
   THR_COMP_INTERINTRA_ZEROA,
   THR_COMP_INTERINTRA_NEARESTA,
   THR_COMP_INTERINTRA_NEARA,
@@ -346,11 +333,7 @@
 #if CONFIG_EXT_REFS
   THR_LAST2,
   THR_LAST3,
-  THR_LAST4,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   THR_BWDR,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_GOLD,
   THR_ALTR,
@@ -359,14 +342,16 @@
 #if CONFIG_EXT_REFS
   THR_COMP_L2A,
   THR_COMP_L3A,
-  THR_COMP_L4A,
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  THR_COMP_LB,
-  THR_COMP_GB,
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   THR_COMP_GA,
+
+#if CONFIG_EXT_REFS
+  THR_COMP_LB,
+  THR_COMP_L2B,
+  THR_COMP_L3B,
+  THR_COMP_GB,
+#endif  // CONFIG_EXT_REFS
+
   THR_INTRA,
 } THR_MODES_SUB8X8;
 
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index b000da8..c7ae00f 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -46,59 +46,42 @@
 
 #if CONFIG_EXT_REFS
 
-#define LAST_FRAME_MODE_MASK    ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << LAST2_FRAME) | (1 << INTRA_FRAME) | \
-                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
-#define LAST2_FRAME_MODE_MASK   ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << LAST_FRAME) | (1 << INTRA_FRAME) | \
-                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
-#define LAST3_FRAME_MODE_MASK   ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << LAST_FRAME) | (1 << INTRA_FRAME) | \
-                                 (1 << LAST2_FRAME) | (1 << LAST4_FRAME))
-#define LAST4_FRAME_MODE_MASK   ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << LAST_FRAME) | (1 << INTRA_FRAME) | \
-                                 (1 << LAST2_FRAME) | (1 << LAST3_FRAME))
-#define GOLDEN_FRAME_MODE_MASK  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << LAST2_FRAME) | (1 << INTRA_FRAME) | \
-                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
-#define ALT_REF_MODE_MASK       ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
-                                 (1 << LAST2_FRAME) | (1 << INTRA_FRAME) | \
-                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
+#define LAST_FRAME_MODE_MASK    ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | \
+                                 (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | \
+                                 (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
+#define LAST2_FRAME_MODE_MASK   ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | \
+                                 (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | \
+                                 (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
+#define LAST3_FRAME_MODE_MASK   ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | \
+                                 (1 << LAST2_FRAME) | (1 << GOLDEN_FRAME) | \
+                                 (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
+#define GOLDEN_FRAME_MODE_MASK  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | \
+                                 (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | \
+                                 (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
+#define BWDREF_FRAME_MODE_MASK  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | \
+                                 (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | \
+                                 (1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME))
+#define ALTREF_FRAME_MODE_MASK  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | \
+                                 (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | \
+                                 (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME))
 
-#else  // CONFIG_EXT_REFS
-
-#if CONFIG_BIDIR_PRED
-
-#define LAST_FRAME_MODE_MASK    ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << BWDREF_FRAME) | (1 << INTRA_FRAME))
-#define GOLDEN_FRAME_MODE_MASK  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | \
-                                 (1 << BWDREF_FRAME) | (1 << INTRA_FRAME))
-#define BWD_REF_MODE_MASK       ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
-                                 (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
-// TODO(zoeliu): To rename the following to ALTREF_MODE_MASK
-#define ALT_REF_MODE_MASK       ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
-                                 (1 << BWDREF_FRAME) | (1 << INTRA_FRAME))
-
-
-#else  // CONFIG_BIDIR_PRED
+#else
 
 #define LAST_FRAME_MODE_MASK    ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
                                  (1 << INTRA_FRAME))
 #define GOLDEN_FRAME_MODE_MASK  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | \
                                  (1 << INTRA_FRAME))
-#define ALT_REF_MODE_MASK       ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
+#define ALTREF_FRAME_MODE_MASK  ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
                                  (1 << INTRA_FRAME))
 
-#endif  // CONFIG_BIDIR_PRED
-
 #endif  // CONFIG_EXT_REFS
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 #define SECOND_REF_FRAME_MASK   ((1 << ALTREF_FRAME) | (1 << BWDREF_FRAME) | \
                                  0x01)
 #else
 #define SECOND_REF_FRAME_MASK   ((1 << ALTREF_FRAME) | 0x01)
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 #define MIN_EARLY_TERM_INDEX    3
 #define NEW_MV_DISCOUNT_FACTOR  8
@@ -143,11 +126,7 @@
 #if CONFIG_EXT_REFS
   {NEARESTMV, {LAST2_FRAME,  NONE}},
   {NEARESTMV, {LAST3_FRAME,  NONE}},
-  {NEARESTMV, {LAST4_FRAME,  NONE}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   {NEARESTMV, {BWDREF_FRAME, NONE}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEARESTMV, {ALTREF_FRAME, NONE}},
   {NEARESTMV, {GOLDEN_FRAME, NONE}},
@@ -158,11 +137,7 @@
 #if CONFIG_EXT_REFS
   {NEWMV,     {LAST2_FRAME,  NONE}},
   {NEWMV,     {LAST3_FRAME,  NONE}},
-  {NEWMV,     {LAST4_FRAME,  NONE}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   {NEWMV,     {BWDREF_FRAME, NONE}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEWMV,     {ALTREF_FRAME, NONE}},
   {NEWMV,     {GOLDEN_FRAME, NONE}},
@@ -171,11 +146,7 @@
 #if CONFIG_EXT_REFS
   {NEARMV,    {LAST2_FRAME,  NONE}},
   {NEARMV,    {LAST3_FRAME,  NONE}},
-  {NEARMV,    {LAST4_FRAME,  NONE}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   {NEARMV,    {BWDREF_FRAME, NONE}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEARMV,    {ALTREF_FRAME, NONE}},
   {NEARMV,    {GOLDEN_FRAME, NONE}},
@@ -185,11 +156,7 @@
 #if CONFIG_EXT_REFS
   {NEWFROMNEARMV,    {LAST2_FRAME,  NONE}},
   {NEWFROMNEARMV,    {LAST3_FRAME,  NONE}},
-  {NEWFROMNEARMV,    {LAST4_FRAME,  NONE}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   {NEWFROMNEARMV,    {BWDREF_FRAME, NONE}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEWFROMNEARMV,    {ALTREF_FRAME, NONE}},
   {NEWFROMNEARMV,    {GOLDEN_FRAME, NONE}},
@@ -199,11 +166,7 @@
 #if CONFIG_EXT_REFS
   {ZEROMV,    {LAST2_FRAME,  NONE}},
   {ZEROMV,    {LAST3_FRAME,  NONE}},
-  {ZEROMV,    {LAST4_FRAME,  NONE}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   {ZEROMV,    {BWDREF_FRAME, NONE}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {ZEROMV,    {GOLDEN_FRAME, NONE}},
   {ZEROMV,    {ALTREF_FRAME, NONE}},
@@ -215,50 +178,44 @@
 #if CONFIG_EXT_REFS
   {NEAREST_NEARESTMV, {LAST2_FRAME,  ALTREF_FRAME}},
   {NEAREST_NEARESTMV, {LAST3_FRAME,  ALTREF_FRAME}},
-  {NEAREST_NEARESTMV, {LAST4_FRAME,  ALTREF_FRAME}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  {NEAREST_NEARESTMV, {LAST_FRAME,   BWDREF_FRAME}},
-  {NEAREST_NEARESTMV, {GOLDEN_FRAME, BWDREF_FRAME}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEAREST_NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
+#if CONFIG_EXT_REFS
+  {NEAREST_NEARESTMV, {LAST_FRAME,   BWDREF_FRAME}},
+  {NEAREST_NEARESTMV, {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEAREST_NEARESTMV, {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEAREST_NEARESTMV, {GOLDEN_FRAME, BWDREF_FRAME}},
+#endif  // CONFIG_EXT_REFS
+
 #else  // CONFIG_EXT_INTER
+
   {NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
 #if CONFIG_EXT_REFS
   {NEARESTMV, {LAST2_FRAME,  ALTREF_FRAME}},
   {NEARESTMV, {LAST3_FRAME,  ALTREF_FRAME}},
-  {NEARESTMV, {LAST4_FRAME,  ALTREF_FRAME}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  {NEARESTMV, {LAST_FRAME,   BWDREF_FRAME}},
-  {NEARESTMV, {GOLDEN_FRAME, BWDREF_FRAME}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
+#if CONFIG_EXT_REFS
+  {NEARESTMV, {LAST_FRAME,   BWDREF_FRAME}},
+  {NEARESTMV, {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEARESTMV, {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEARESTMV, {GOLDEN_FRAME, BWDREF_FRAME}},
+#endif  // CONFIG_EXT_REFS
 #endif  // CONFIG_EXT_INTER
 
   {TM_PRED,   {INTRA_FRAME,  NONE}},
 
 #if CONFIG_EXT_INTER
   {NEAR_NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
-  {NEAR_NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEAREST_NEARMV, {LAST_FRAME,   ALTREF_FRAME}},
-  {NEAREST_NEARMV, {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEAR_NEARMV,    {LAST_FRAME,   ALTREF_FRAME}},
-  {NEAR_NEARMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEW_NEARESTMV,  {LAST_FRAME,   ALTREF_FRAME}},
-  {NEW_NEARESTMV,  {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEAREST_NEWMV,  {LAST_FRAME,   ALTREF_FRAME}},
-  {NEAREST_NEWMV,  {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEW_NEARMV,     {LAST_FRAME,   ALTREF_FRAME}},
-  {NEW_NEARMV,     {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEAR_NEWMV,     {LAST_FRAME,   ALTREF_FRAME}},
-  {NEAR_NEWMV,     {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEW_NEWMV,      {LAST_FRAME,   ALTREF_FRAME}},
-  {NEW_NEWMV,      {GOLDEN_FRAME, ALTREF_FRAME}},
   {ZERO_ZEROMV,    {LAST_FRAME,   ALTREF_FRAME}},
-  {ZERO_ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
+
 #if CONFIG_EXT_REFS
   {NEAR_NEARESTMV, {LAST2_FRAME,  ALTREF_FRAME}},
   {NEAREST_NEARMV, {LAST2_FRAME,  ALTREF_FRAME}},
@@ -279,35 +236,58 @@
   {NEAR_NEWMV,     {LAST3_FRAME,  ALTREF_FRAME}},
   {NEW_NEWMV,      {LAST3_FRAME,  ALTREF_FRAME}},
   {ZERO_ZEROMV,    {LAST3_FRAME,  ALTREF_FRAME}},
+#endif  // CONFIG_EXT_REFS
 
-  {NEAR_NEARESTMV, {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEAREST_NEARMV, {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEAR_NEARMV,    {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEW_NEARESTMV,  {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEAREST_NEWMV,  {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEW_NEARMV,     {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEAR_NEWMV,     {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEW_NEWMV,      {LAST4_FRAME,  ALTREF_FRAME}},
-  {ZERO_ZEROMV,    {LAST4_FRAME,  ALTREF_FRAME}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
+  {NEAR_NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEAREST_NEARMV, {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEAR_NEARMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEW_NEARESTMV,  {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEAREST_NEWMV,  {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEW_NEARMV,     {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEAR_NEWMV,     {GOLDEN_FRAME, ALTREF_FRAME}},
+  {NEW_NEWMV,      {GOLDEN_FRAME, ALTREF_FRAME}},
+  {ZERO_ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
+
+#if CONFIG_EXT_REFS
   {NEAR_NEARESTMV, {LAST_FRAME,   BWDREF_FRAME}},
-  {NEAR_NEARESTMV, {GOLDEN_FRAME, BWDREF_FRAME}},
   {NEAREST_NEARMV, {LAST_FRAME,   BWDREF_FRAME}},
-  {NEAREST_NEARMV, {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEAR_NEARMV,    {LAST_FRAME,   BWDREF_FRAME}},
   {NEW_NEARESTMV,  {LAST_FRAME,   BWDREF_FRAME}},
-  {NEW_NEARESTMV,  {GOLDEN_FRAME, BWDREF_FRAME}},
   {NEAREST_NEWMV,  {LAST_FRAME,   BWDREF_FRAME}},
-  {NEAREST_NEWMV,  {GOLDEN_FRAME, BWDREF_FRAME}},
   {NEW_NEARMV,     {LAST_FRAME,   BWDREF_FRAME}},
-  {NEW_NEARMV,     {GOLDEN_FRAME, BWDREF_FRAME}},
   {NEAR_NEWMV,     {LAST_FRAME,   BWDREF_FRAME}},
-  {NEAR_NEWMV,     {GOLDEN_FRAME, BWDREF_FRAME}},
   {NEW_NEWMV,      {LAST_FRAME,   BWDREF_FRAME}},
-  {NEW_NEWMV,      {GOLDEN_FRAME, BWDREF_FRAME}},
   {ZERO_ZEROMV,    {LAST_FRAME,   BWDREF_FRAME}},
+
+  {NEAR_NEARESTMV, {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEAREST_NEARMV, {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEAR_NEARMV,    {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEW_NEARESTMV,  {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEAREST_NEWMV,  {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEW_NEARMV,     {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEAR_NEWMV,     {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEW_NEWMV,      {LAST2_FRAME,  BWDREF_FRAME}},
+  {ZERO_ZEROMV,    {LAST2_FRAME,  BWDREF_FRAME}},
+
+  {NEAR_NEARESTMV, {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEAREST_NEARMV, {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEAR_NEARMV,    {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEW_NEARESTMV,  {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEAREST_NEWMV,  {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEW_NEARMV,     {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEAR_NEWMV,     {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEW_NEWMV,      {LAST3_FRAME,  BWDREF_FRAME}},
+  {ZERO_ZEROMV,    {LAST3_FRAME,  BWDREF_FRAME}},
+
+  {NEAR_NEARESTMV, {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEAREST_NEARMV, {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEAR_NEARMV,    {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEW_NEARESTMV,  {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEAREST_NEWMV,  {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEW_NEARMV,     {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEAR_NEWMV,     {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEW_NEWMV,      {GOLDEN_FRAME, BWDREF_FRAME}},
   {ZERO_ZEROMV,    {GOLDEN_FRAME, BWDREF_FRAME}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 
 #else  // CONFIG_EXT_INTER
@@ -319,31 +299,35 @@
   {NEWMV,     {LAST2_FRAME,  ALTREF_FRAME}},
   {NEARMV,    {LAST3_FRAME,  ALTREF_FRAME}},
   {NEWMV,     {LAST3_FRAME,  ALTREF_FRAME}},
-  {NEARMV,    {LAST4_FRAME,  ALTREF_FRAME}},
-  {NEWMV,     {LAST4_FRAME,  ALTREF_FRAME}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  {NEARMV,    {LAST_FRAME,   BWDREF_FRAME}},
-  {NEWMV,     {LAST_FRAME,   BWDREF_FRAME}},
-  {NEARMV,    {GOLDEN_FRAME, BWDREF_FRAME}},
-  {NEWMV,     {GOLDEN_FRAME, BWDREF_FRAME}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {NEARMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
   {NEWMV,     {GOLDEN_FRAME, ALTREF_FRAME}},
 
+#if CONFIG_EXT_REFS
+  {NEARMV,    {LAST_FRAME,   BWDREF_FRAME}},
+  {NEWMV,     {LAST_FRAME,   BWDREF_FRAME}},
+  {NEARMV,    {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEWMV,     {LAST2_FRAME,  BWDREF_FRAME}},
+  {NEARMV,    {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEWMV,     {LAST3_FRAME,  BWDREF_FRAME}},
+  {NEARMV,    {GOLDEN_FRAME, BWDREF_FRAME}},
+  {NEWMV,     {GOLDEN_FRAME, BWDREF_FRAME}},
+#endif  // CONFIG_EXT_REFS
+
   {ZEROMV,    {LAST_FRAME,   ALTREF_FRAME}},
 #if CONFIG_EXT_REFS
-  {ZEROMV,    {LAST3_FRAME,  ALTREF_FRAME}},
   {ZEROMV,    {LAST2_FRAME,  ALTREF_FRAME}},
-  {ZEROMV,    {LAST4_FRAME,  ALTREF_FRAME}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  {ZEROMV,    {LAST_FRAME,   BWDREF_FRAME}},
-  {ZEROMV,    {GOLDEN_FRAME, BWDREF_FRAME}},
-#endif  // CONFIG_BIDIR_PRED
+  {ZEROMV,    {LAST3_FRAME,  ALTREF_FRAME}},
 #endif  // CONFIG_EXT_REFS
   {ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
+
+#if CONFIG_EXT_REFS
+  {ZEROMV,    {LAST_FRAME,   BWDREF_FRAME}},
+  {ZEROMV,    {LAST2_FRAME,  BWDREF_FRAME}},
+  {ZEROMV,    {LAST3_FRAME,  BWDREF_FRAME}},
+  {ZEROMV,    {GOLDEN_FRAME, BWDREF_FRAME}},
+#endif  // CONFIG_EXT_REFS
+
 #endif  // CONFIG_EXT_INTER
 
   {H_PRED,    {INTRA_FRAME,  NONE}},
@@ -371,11 +355,6 @@
   {NEARESTMV, {LAST3_FRAME,  INTRA_FRAME}},
   {NEARMV,    {LAST3_FRAME,  INTRA_FRAME}},
   {NEWMV,     {LAST3_FRAME,  INTRA_FRAME}},
-
-  {ZEROMV,    {LAST4_FRAME,  INTRA_FRAME}},
-  {NEARESTMV, {LAST4_FRAME,  INTRA_FRAME}},
-  {NEARMV,    {LAST4_FRAME,  INTRA_FRAME}},
-  {NEWMV,     {LAST4_FRAME,  INTRA_FRAME}},
 #endif  // CONFIG_EXT_REFS
 
   {ZEROMV,    {GOLDEN_FRAME, INTRA_FRAME}},
@@ -383,6 +362,13 @@
   {NEARMV,    {GOLDEN_FRAME, INTRA_FRAME}},
   {NEWMV,     {GOLDEN_FRAME, INTRA_FRAME}},
 
+#if CONFIG_EXT_REFS
+  {ZEROMV,    {BWDREF_FRAME, INTRA_FRAME}},
+  {NEARESTMV, {BWDREF_FRAME, INTRA_FRAME}},
+  {NEARMV,    {BWDREF_FRAME, INTRA_FRAME}},
+  {NEWMV,     {BWDREF_FRAME, INTRA_FRAME}},
+#endif  // CONFIG_EXT_REFS
+
   {ZEROMV,    {ALTREF_FRAME, INTRA_FRAME}},
   {NEARESTMV, {ALTREF_FRAME, INTRA_FRAME}},
   {NEARMV,    {ALTREF_FRAME, INTRA_FRAME}},
@@ -395,11 +381,7 @@
 #if CONFIG_EXT_REFS
   {{LAST2_FRAME,  NONE}},
   {{LAST3_FRAME,  NONE}},
-  {{LAST4_FRAME,  NONE}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
   {{BWDREF_FRAME, NONE}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {{GOLDEN_FRAME, NONE}},
   {{ALTREF_FRAME, NONE}},
@@ -408,14 +390,16 @@
 #if CONFIG_EXT_REFS
   {{LAST2_FRAME,  ALTREF_FRAME}},
   {{LAST3_FRAME,  ALTREF_FRAME}},
-  {{LAST4_FRAME,  ALTREF_FRAME}},
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
-  {{LAST_FRAME,   BWDREF_FRAME}},
-  {{GOLDEN_FRAME, BWDREF_FRAME}},
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
   {{GOLDEN_FRAME, ALTREF_FRAME}},
+
+#if CONFIG_EXT_REFS
+  {{LAST_FRAME,   BWDREF_FRAME}},
+  {{LAST2_FRAME,  BWDREF_FRAME}},
+  {{LAST3_FRAME,  BWDREF_FRAME}},
+  {{GOLDEN_FRAME, BWDREF_FRAME}},
+#endif  // CONFIG_EXT_REFS
+
   {{INTRA_FRAME,  NONE}},
 };
 
@@ -5834,24 +5818,19 @@
     if (cm->reference_mode != COMPOUND_REFERENCE) {
       vpx_prob ref_single_p1 = vp10_get_pred_prob_single_ref_p1(cm, xd);
       vpx_prob ref_single_p2 = vp10_get_pred_prob_single_ref_p2(cm, xd);
-#if CONFIG_EXT_REFS || CONFIG_BIDIR_PRED
-      vpx_prob ref_single_p3 = vp10_get_pred_prob_single_ref_p3(cm, xd);
-#endif  // CONFIG_EXT_REFS || CONFIG_BIDIR_PRED
 #if CONFIG_EXT_REFS
+      vpx_prob ref_single_p3 = vp10_get_pred_prob_single_ref_p3(cm, xd);
       vpx_prob ref_single_p4 = vp10_get_pred_prob_single_ref_p4(cm, xd);
       vpx_prob ref_single_p5 = vp10_get_pred_prob_single_ref_p5(cm, xd);
 #endif  // CONFIG_EXT_REFS
+
       unsigned int base_cost = vp10_cost_bit(intra_inter_p, 1);
 
       ref_costs_single[LAST_FRAME] =
 #if CONFIG_EXT_REFS
           ref_costs_single[LAST2_FRAME] =
           ref_costs_single[LAST3_FRAME] =
-          ref_costs_single[LAST4_FRAME] =
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
           ref_costs_single[BWDREF_FRAME] =
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
           ref_costs_single[GOLDEN_FRAME] =
           ref_costs_single[ALTREF_FRAME] = base_cost;
@@ -5860,44 +5839,37 @@
       ref_costs_single[LAST_FRAME]   += vp10_cost_bit(ref_single_p1, 0);
       ref_costs_single[LAST2_FRAME]  += vp10_cost_bit(ref_single_p1, 0);
       ref_costs_single[LAST3_FRAME]  += vp10_cost_bit(ref_single_p1, 0);
-      ref_costs_single[LAST4_FRAME]  += vp10_cost_bit(ref_single_p1, 0);
-      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p1, 1);
+      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p1, 0);
+      ref_costs_single[BWDREF_FRAME] += vp10_cost_bit(ref_single_p1, 1);
       ref_costs_single[ALTREF_FRAME] += vp10_cost_bit(ref_single_p1, 1);
 
       ref_costs_single[LAST_FRAME]   += vp10_cost_bit(ref_single_p3, 0);
       ref_costs_single[LAST2_FRAME]  += vp10_cost_bit(ref_single_p3, 0);
       ref_costs_single[LAST3_FRAME]  += vp10_cost_bit(ref_single_p3, 1);
-      ref_costs_single[LAST4_FRAME]  += vp10_cost_bit(ref_single_p3, 1);
-      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p2, 0);
+      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p3, 1);
+
+      ref_costs_single[BWDREF_FRAME] += vp10_cost_bit(ref_single_p2, 0);
       ref_costs_single[ALTREF_FRAME] += vp10_cost_bit(ref_single_p2, 1);
 
       ref_costs_single[LAST_FRAME]   += vp10_cost_bit(ref_single_p4, 0);
       ref_costs_single[LAST2_FRAME]  += vp10_cost_bit(ref_single_p4, 1);
+
       ref_costs_single[LAST3_FRAME]  += vp10_cost_bit(ref_single_p5, 0);
-      ref_costs_single[LAST4_FRAME]  += vp10_cost_bit(ref_single_p5, 1);
-#else  // CONFIG_EXT_REFS
+      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p5, 1);
+#else
       ref_costs_single[LAST_FRAME]   += vp10_cost_bit(ref_single_p1, 0);
       ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p1, 1);
-      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p2, 0);
       ref_costs_single[ALTREF_FRAME] += vp10_cost_bit(ref_single_p1, 1);
+
+      ref_costs_single[GOLDEN_FRAME] += vp10_cost_bit(ref_single_p2, 0);
       ref_costs_single[ALTREF_FRAME] += vp10_cost_bit(ref_single_p2, 1);
-#if CONFIG_BIDIR_PRED
-      ref_costs_single[ALTREF_FRAME] += vp10_cost_bit(ref_single_p3, 1);
-      ref_costs_single[BWDREF_FRAME] += vp10_cost_bit(ref_single_p1, 1);
-      ref_costs_single[BWDREF_FRAME] += vp10_cost_bit(ref_single_p2, 1);
-      ref_costs_single[BWDREF_FRAME] += vp10_cost_bit(ref_single_p3, 0);
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
     } else {
       ref_costs_single[LAST_FRAME]   = 512;
 #if CONFIG_EXT_REFS
       ref_costs_single[LAST2_FRAME]  = 512;
       ref_costs_single[LAST3_FRAME]  = 512;
-      ref_costs_single[LAST4_FRAME]  = 512;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
       ref_costs_single[BWDREF_FRAME] = 512;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
       ref_costs_single[GOLDEN_FRAME] = 512;
       ref_costs_single[ALTREF_FRAME] = 512;
@@ -5908,11 +5880,7 @@
 #if CONFIG_EXT_REFS
       vpx_prob ref_comp_p1 = vp10_get_pred_prob_comp_ref_p1(cm, xd);
       vpx_prob ref_comp_p2 = vp10_get_pred_prob_comp_ref_p2(cm, xd);
-      vpx_prob ref_comp_p3 = vp10_get_pred_prob_comp_ref_p3(cm, xd);
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
       vpx_prob bwdref_comp_p = vp10_get_pred_prob_comp_bwdref_p(cm, xd);
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
 
       unsigned int base_cost = vp10_cost_bit(intra_inter_p, 1);
@@ -5921,50 +5889,40 @@
 #if CONFIG_EXT_REFS
           ref_costs_comp[LAST2_FRAME] =
           ref_costs_comp[LAST3_FRAME] =
-          ref_costs_comp[LAST4_FRAME] =
 #endif  // CONFIG_EXT_REFS
           ref_costs_comp[GOLDEN_FRAME] = base_cost;
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-      // NOTE(zoeliu): BWDREF and ALTREF each add an extra cost by coding 1
-      //               more bit.
+#if CONFIG_EXT_REFS
       ref_costs_comp[BWDREF_FRAME] = ref_costs_comp[ALTREF_FRAME] = 0;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 #if CONFIG_EXT_REFS
       ref_costs_comp[LAST_FRAME]   += vp10_cost_bit(ref_comp_p, 0);
       ref_costs_comp[LAST2_FRAME]  += vp10_cost_bit(ref_comp_p, 0);
       ref_costs_comp[LAST3_FRAME]  += vp10_cost_bit(ref_comp_p, 1);
-      ref_costs_comp[LAST4_FRAME]  += vp10_cost_bit(ref_comp_p, 1);
       ref_costs_comp[GOLDEN_FRAME] += vp10_cost_bit(ref_comp_p, 1);
 
       ref_costs_comp[LAST_FRAME]   += vp10_cost_bit(ref_comp_p1, 1);
       ref_costs_comp[LAST2_FRAME]  += vp10_cost_bit(ref_comp_p1, 0);
+
       ref_costs_comp[LAST3_FRAME]  += vp10_cost_bit(ref_comp_p2, 0);
-      ref_costs_comp[LAST4_FRAME]  += vp10_cost_bit(ref_comp_p2, 0);
       ref_costs_comp[GOLDEN_FRAME] += vp10_cost_bit(ref_comp_p2, 1);
 
-      ref_costs_comp[LAST3_FRAME]  += vp10_cost_bit(ref_comp_p3, 1);
-      ref_costs_comp[LAST4_FRAME]  += vp10_cost_bit(ref_comp_p3, 0);
-#else  // CONFIG_EXT_REFS
-      ref_costs_comp[LAST_FRAME]   += vp10_cost_bit(ref_comp_p, 0);
-      ref_costs_comp[GOLDEN_FRAME] += vp10_cost_bit(ref_comp_p, 1);
-#if CONFIG_BIDIR_PRED
+      // NOTE(zoeliu): BWDREF and ALTREF each add an extra cost by coding 1
+      //               more bit.
       ref_costs_comp[BWDREF_FRAME] += vp10_cost_bit(bwdref_comp_p, 0);
       ref_costs_comp[ALTREF_FRAME] += vp10_cost_bit(bwdref_comp_p, 1);
-#endif  // CONFIG_BIDIR_PRED
+#else
+      ref_costs_comp[LAST_FRAME]   += vp10_cost_bit(ref_comp_p, 0);
+      ref_costs_comp[GOLDEN_FRAME] += vp10_cost_bit(ref_comp_p, 1);
 #endif  // CONFIG_EXT_REFS
     } else {
       ref_costs_comp[LAST_FRAME]   = 512;
 #if CONFIG_EXT_REFS
       ref_costs_comp[LAST2_FRAME]  = 512;
       ref_costs_comp[LAST3_FRAME]  = 512;
-      ref_costs_comp[LAST4_FRAME]  = 512;
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
       ref_costs_comp[BWDREF_FRAME] = 512;
       ref_costs_comp[ALTREF_FRAME] = 512;
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
       ref_costs_comp[GOLDEN_FRAME] = 512;
     }
@@ -8537,12 +8495,11 @@
 #if CONFIG_EXT_REFS
     VP9_LAST2_FLAG,
     VP9_LAST3_FLAG,
-    VP9_LAST4_FLAG,
 #endif  // CONFIG_EXT_REFS
     VP9_GOLD_FLAG,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     VP9_BWD_FLAG,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     VP9_ALT_FLAG
   };
   int64_t best_rd = best_rd_so_far;
@@ -8722,17 +8679,17 @@
       // Skip checking missing references in both single and compound reference
       // modes. Note that a mode will be skipped iff both reference frames
       // are masked out.
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       if (ref_frame == BWDREF_FRAME || ref_frame == ALTREF_FRAME) {
         ref_frame_skip_mask[0] |= (1 << ref_frame);
         ref_frame_skip_mask[1] |= ((1 << ref_frame) | 0x01);
       } else {
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
         ref_frame_skip_mask[0] |= (1 << ref_frame);
         ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     } else {
       for (i = LAST_FRAME; i <= ALTREF_FRAME; ++i) {
         // Skip fixed mv modes for poor references
@@ -8765,11 +8722,7 @@
 #if CONFIG_EXT_REFS
           (1 << LAST2_FRAME) |
           (1 << LAST3_FRAME) |
-          (1 << LAST4_FRAME) |
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
           (1 << BWDREF_FRAME) |
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
           (1 << GOLDEN_FRAME);
       ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;
@@ -8952,26 +8905,22 @@
           ref_frame_skip_mask[0] |= LAST3_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-        case LAST4_FRAME:
-          ref_frame_skip_mask[0] |= LAST4_FRAME_MODE_MASK;
-          ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
-          break;
 #endif  // CONFIG_EXT_REFS
         case GOLDEN_FRAME:
           ref_frame_skip_mask[0] |= GOLDEN_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
         case BWDREF_FRAME:
-          ref_frame_skip_mask[0] |= BWD_REF_MODE_MASK;
+          ref_frame_skip_mask[0] |= BWDREF_FRAME_MODE_MASK;
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
           break;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
         case ALTREF_FRAME:
-          ref_frame_skip_mask[0] |= ALT_REF_MODE_MASK;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+          ref_frame_skip_mask[0] |= ALTREF_FRAME_MODE_MASK;
+#if CONFIG_EXT_REFS
           ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
           break;
         case NONE:
         case MAX_REF_FRAMES:
@@ -10313,12 +10262,11 @@
 #if CONFIG_EXT_REFS
     VP9_LAST2_FLAG,
     VP9_LAST3_FLAG,
-    VP9_LAST4_FLAG,
 #endif  // CONFIG_EXT_REFS
     VP9_GOLD_FLAG,
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
     VP9_BWD_FLAG,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
     VP9_ALT_FLAG
   };
   int64_t best_rd = best_rd_so_far;
@@ -10446,11 +10394,7 @@
 #if CONFIG_EXT_REFS
                                       (1 << LAST2_FRAME) |
                                       (1 << LAST3_FRAME) |
-                                      (1 << LAST4_FRAME) |
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
                                       (1 << BWDREF_FRAME) |
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
                                       (1 << ALTREF_FRAME);
             ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
@@ -10459,24 +10403,16 @@
           case LAST2_FRAME:
             ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
                                       (1 << LAST3_FRAME) |
-                                      (1 << LAST4_FRAME) |
                                       (1 << GOLDEN_FRAME) |
+                                      (1 << BWDREF_FRAME) |
                                       (1 << ALTREF_FRAME);
             ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
             break;
           case LAST3_FRAME:
             ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
                                       (1 << LAST2_FRAME) |
-                                      (1 << LAST4_FRAME) |
                                       (1 << GOLDEN_FRAME) |
-                                      (1 << ALTREF_FRAME);
-            ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
-            break;
-          case LAST4_FRAME:
-            ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
-                                      (1 << LAST2_FRAME) |
-                                      (1 << LAST3_FRAME) |
-                                      (1 << GOLDEN_FRAME) |
+                                      (1 << BWDREF_FRAME) |
                                       (1 << ALTREF_FRAME);
             ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
             break;
@@ -10486,38 +10422,32 @@
 #if CONFIG_EXT_REFS
                                       (1 << LAST2_FRAME) |
                                       (1 << LAST3_FRAME) |
-                                      (1 << LAST4_FRAME) |
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
                                       (1 << BWDREF_FRAME) |
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
                                       (1 << ALTREF_FRAME);
             ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
             break;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
           case BWDREF_FRAME:
             ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
+                                      (1 << LAST2_FRAME) |
+                                      (1 << LAST3_FRAME) |
                                       (1 << GOLDEN_FRAME) |
                                       (1 << ALTREF_FRAME);
-            ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
+            ref_frame_skip_mask[1] |= (1 << ALTREF_FRAME) | 0x01;
             break;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
           case ALTREF_FRAME:
-            ref_frame_skip_mask[0] |= (1 << GOLDEN_FRAME) |
+            ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
 #if CONFIG_EXT_REFS
                                       (1 << LAST2_FRAME) |
                                       (1 << LAST3_FRAME) |
-                                      (1 << LAST4_FRAME) |
-#else  // CONFIG_EXT_REFS
-#if CONFIG_BIDIR_PRED
                                       (1 << BWDREF_FRAME) |
-#endif  // CONFIG_BIDIR_PRED
 #endif  // CONFIG_EXT_REFS
-                                      (1 << LAST_FRAME);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
-            ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+                                      (1 << GOLDEN_FRAME);
+#if CONFIG_EXT_REFS
+            ref_frame_skip_mask[1] |= (1 << BWDREF_FRAME) | 0x01;
+#endif  // CONFIG_EXT_REFS
             break;
           case NONE:
           case MAX_REF_FRAMES:
@@ -10671,11 +10601,13 @@
           rd_opt->threshes[segment_id][bsize][THR_LAST2] : this_rd_thresh;
       this_rd_thresh = (ref_frame == LAST3_FRAME) ?
           rd_opt->threshes[segment_id][bsize][THR_LAST3] : this_rd_thresh;
-      this_rd_thresh = (ref_frame == LAST4_FRAME) ?
-          rd_opt->threshes[segment_id][bsize][THR_LAST4] : this_rd_thresh;
 #endif  // CONFIG_EXT_REFS
       this_rd_thresh = (ref_frame == GOLDEN_FRAME) ?
           rd_opt->threshes[segment_id][bsize][THR_GOLD] : this_rd_thresh;
+#if CONFIG_EXT_REFS
+      // TODO(zoeliu): To explore whether this_rd_thresh should consider
+      //               BWDREF_FRAME and ALTREF_FRAME
+#endif  // CONFIG_EXT_REFS
 
       // TODO(any): Add search of the tx_type to improve rd performance at the
       // expense of speed.
diff --git a/vp10/vp10_cx_iface.c b/vp10/vp10_cx_iface.c
index bed6648..a707d11 100644
--- a/vp10/vp10_cx_iface.c
+++ b/vp10/vp10_cx_iface.c
@@ -25,9 +25,9 @@
 struct vp10_extracfg {
   int                         cpu_used;  // available cpu percentage in 1/16
   unsigned int                enable_auto_alt_ref;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   unsigned int                enable_auto_bwd_ref;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   unsigned int                noise_sensitivity;
   unsigned int                sharpness;
   unsigned int                static_thresh;
@@ -58,9 +58,9 @@
 static struct vp10_extracfg default_extra_cfg = {
   0,                            // cpu_used
   1,                            // enable_auto_alt_ref
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   0,                            // enable_auto_bwd_ref
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   0,                            // noise_sensitivity
   0,                            // sharpness
   0,                            // static_thresh
@@ -205,9 +205,9 @@
           "or kf_max_dist instead.");
 
   RANGE_CHECK(extra_cfg, enable_auto_alt_ref, 0, 2);
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   RANGE_CHECK(extra_cfg, enable_auto_bwd_ref, 0, 2);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   RANGE_CHECK(extra_cfg, cpu_used, -8, 8);
   RANGE_CHECK_HI(extra_cfg, noise_sensitivity, 6);
   RANGE_CHECK(extra_cfg, superblock_size,
@@ -420,9 +420,9 @@
   oxcf->speed                  =  abs(extra_cfg->cpu_used);
   oxcf->encode_breakout        =  extra_cfg->static_thresh;
   oxcf->enable_auto_arf        =  extra_cfg->enable_auto_alt_ref;
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   oxcf->enable_auto_brf        =  extra_cfg->enable_auto_bwd_ref;
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   oxcf->noise_sensitivity      =  extra_cfg->noise_sensitivity;
   oxcf->sharpness              =  extra_cfg->sharpness;
 
@@ -586,14 +586,14 @@
   return update_extra_cfg(ctx, &extra_cfg);
 }
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 static vpx_codec_err_t ctrl_set_enable_auto_bwd_ref(vpx_codec_alg_priv_t *ctx,
                                                     va_list args) {
   struct vp10_extracfg extra_cfg = ctx->extra_cfg;
   extra_cfg.enable_auto_bwd_ref = CAST(VP8E_SET_ENABLEAUTOBWDREF, args);
   return update_extra_cfg(ctx, &extra_cfg);
 }
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 static vpx_codec_err_t ctrl_set_noise_sensitivity(vpx_codec_alg_priv_t *ctx,
                                                   va_list args) {
@@ -943,14 +943,14 @@
     // TODO(jzern) the checks related to cpi's validity should be treated as a
     // failure condition, encoder setup is done fully in init() currently.
     if (res == VPX_CODEC_OK) {
-      // There's no codec control for multiple alt-refs so check the encoder
-      // instance for its status to determine the compressed data size.
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
       data_sz = ctx->cfg.g_w * ctx->cfg.g_h * get_image_bps(img);
 #else
+      // There's no codec control for multiple alt-refs so check the encoder
+      // instance for its status to determine the compressed data size.
       data_sz = ctx->cfg.g_w * ctx->cfg.g_h * get_image_bps(img) / 8 *
                 (cpi->multi_arf_allowed ? 8 : 2);
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
       if (data_sz < 4096)
         data_sz = 4096;
       if (ctx->cx_data == NULL || ctx->cx_data_sz < data_sz) {
@@ -1342,9 +1342,9 @@
   {VP8E_SET_SCALEMODE,                ctrl_set_scale_mode},
   {VP8E_SET_CPUUSED,                  ctrl_set_cpuused},
   {VP8E_SET_ENABLEAUTOALTREF,         ctrl_set_enable_auto_alt_ref},
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   {VP8E_SET_ENABLEAUTOBWDREF,         ctrl_set_enable_auto_bwd_ref},
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
   {VP8E_SET_SHARPNESS,                ctrl_set_sharpness},
   {VP8E_SET_STATIC_THRESHOLD,         ctrl_set_static_thresh},
   {VP9E_SET_TILE_COLUMNS,             ctrl_set_tile_columns},
diff --git a/vpx/vp8cx.h b/vpx/vp8cx.h
index 4d9a2a7..710fb51 100644
--- a/vpx/vp8cx.h
+++ b/vpx/vp8cx.h
@@ -184,14 +184,14 @@
    */
   VP8E_SET_ENABLEAUTOALTREF,
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
   /*!\brief Codec control function to enable automatic set and use
    * bwd-pred frames.
    *
    * Supported in codecs: VP10
    */
   VP8E_SET_ENABLEAUTOBWDREF,
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
   /*!\brief control function to set noise sensitivity
    *
@@ -754,10 +754,10 @@
 VPX_CTRL_USE_TYPE(VP8E_SET_ENABLEAUTOALTREF,   unsigned int)
 #define VPX_CTRL_VP8E_SET_ENABLEAUTOALTREF
 
-#if !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#if CONFIG_EXT_REFS
 VPX_CTRL_USE_TYPE(VP8E_SET_ENABLEAUTOBWDREF,   unsigned int)
 #define VPX_CTRL_VP8E_SET_ENABLEAUTOBWDREF
-#endif  // !CONFIG_EXT_REFS && CONFIG_BIDIR_PRED
+#endif  // CONFIG_EXT_REFS
 
 VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY,  unsigned int)
 #define VPX_CTRL_VP8E_SET_NOISE_SENSITIVITY