diff --git a/av1/common/convolve.c b/av1/common/convolve.c
index ab24400..936fb59 100644
--- a/av1/common/convolve.c
+++ b/av1/common/convolve.c
@@ -43,7 +43,7 @@
       for (k = 0; k < filter_size; ++k) sum += src_x[k] * x_filter[k];
 
       sum = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS));
-      if (conv_params->ref)
+      if (conv_params->do_average)
         dst[x] = ROUND_POWER_OF_TWO(dst[x] + sum, 1);
       else
         dst[x] = sum;
@@ -75,7 +75,7 @@
         sum += src_y[k * src_stride] * y_filter[k];
 
       sum = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS));
-      if (conv_params->ref)
+      if (conv_params->do_average)
         dst[y * dst_stride] = ROUND_POWER_OF_TWO(dst[y * dst_stride] + sum, 1);
       else
         dst[y * dst_stride] = sum;
@@ -91,7 +91,7 @@
                           int dst_stride, int w, int h,
                           ConvolveParams *conv_params) {
   assert(conv_params->round == CONVOLVE_OPT_ROUND);
-  if (conv_params->ref == 0) {
+  if (conv_params->do_average == 0) {
     int r;
     for (r = 0; r < h; ++r) {
       memcpy(dst, src, w);
@@ -119,7 +119,7 @@
   if (filter_params.taps == SUBPEL_TAPS) {
     const int16_t *filter_x =
         av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q4);
-    if (conv_params->ref == 0)
+    if (conv_params->do_average == 0)
       aom_convolve8_horiz(src, src_stride, dst, dst_stride, filter_x, x_step_q4,
                           NULL, -1, w, h);
     else
@@ -140,7 +140,7 @@
   if (filter_params.taps == SUBPEL_TAPS) {
     const int16_t *filter_x =
         av1_get_interp_filter_subpel_kernel(filter_params, subpel_x_q4);
-    if (conv_params->ref == 0)
+    if (conv_params->do_average == 0)
       aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x,
                             x_step_q4, NULL, -1, w, h);
     else
@@ -161,7 +161,7 @@
   if (filter_params.taps == SUBPEL_TAPS) {
     const int16_t *filter_y =
         av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q4);
-    if (conv_params->ref == 0) {
+    if (conv_params->do_average == 0) {
       aom_convolve8_vert(src, src_stride, dst, dst_stride, NULL, -1, filter_y,
                          y_step_q4, w, h);
     } else {
@@ -183,7 +183,7 @@
   if (filter_params.taps == SUBPEL_TAPS) {
     const int16_t *filter_y =
         av1_get_interp_filter_subpel_kernel(filter_params, subpel_y_q4);
-    if (conv_params->ref == 0) {
+    if (conv_params->do_average == 0) {
       aom_convolve8_vert_c(src, src_stride, dst, dst_stride, NULL, -1, filter_y,
                            y_step_q4, w, h);
     } else {
@@ -545,6 +545,7 @@
       int temp_stride = max_intermediate_size;
       ConvolveParams temp_conv_params;
       temp_conv_params.ref = 0;
+      temp_conv_params.do_average = 0;
       temp_conv_params.round = CONVOLVE_OPT_ROUND;
       filter_params = filter_params_y;
       filter_size = filter_params_x.taps;
@@ -569,6 +570,7 @@
       int temp_stride = MAX_SB_SIZE;
       ConvolveParams temp_conv_params;
       temp_conv_params.ref = 0;
+      temp_conv_params.do_average = 0;
       temp_conv_params.round = CONVOLVE_OPT_ROUND;
 #if CONFIG_DUAL_FILTER
       filter_params = filter_params_x;
diff --git a/av1/common/convolve.h b/av1/common/convolve.h
index 48b476e..c2c5a37 100644
--- a/av1/common/convolve.h
+++ b/av1/common/convolve.h
@@ -27,6 +27,7 @@
 
 typedef struct ConvolveParams {
   int ref;
+  int do_average;
   CONVOLVE_OPT round;
   CONV_BUF_TYPE *dst;
   int dst_stride;
@@ -36,9 +37,11 @@
   int do_post_rounding;
 } ConvolveParams;
 
-static INLINE ConvolveParams get_conv_params(int ref, int plane) {
+static INLINE ConvolveParams get_conv_params(int ref, int do_average,
+                                             int plane) {
   ConvolveParams conv_params;
   conv_params.ref = ref;
+  conv_params.do_average = do_average;
   conv_params.round = CONVOLVE_OPT_ROUND;
   conv_params.plane = plane;
   conv_params.do_post_rounding = 0;
@@ -60,11 +63,12 @@
                             const int subpel_y_q4, int y_step_q4,
                             ConvolveParams *conv_params);
 
-static INLINE ConvolveParams get_conv_params_no_round(int ref, int plane,
-                                                      int32_t *dst,
+static INLINE ConvolveParams get_conv_params_no_round(int ref, int do_average,
+                                                      int plane, int32_t *dst,
                                                       int dst_stride) {
   ConvolveParams conv_params;
   conv_params.ref = ref;
+  conv_params.do_average = do_average;
   conv_params.round = CONVOLVE_OPT_NO_ROUND;
 #if CONFIG_COMPOUND_ROUND
   conv_params.round_0 = FILTER_BITS;
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c
index fa37a99..6b7a080 100644
--- a/av1/common/reconinter.c
+++ b/av1/common/reconinter.c
@@ -722,16 +722,7 @@
 #endif  // CONFIG_COMPOUND_SEGMENT
     mi->mbmi.interinter_compound_type
   };
-// The prediction filter types used here should be those for
-// the second reference block.
-#if CONFIG_DUAL_FILTER
-  InterpFilter tmp_ipf[4] = {
-    interp_filter[2], interp_filter[3], interp_filter[2], interp_filter[3],
-  };
-#else
-  InterpFilter tmp_ipf = interp_filter;
-#endif  // CONFIG_DUAL_FILTER
-  ConvolveParams conv_params = get_conv_params(0, plane);
+  ConvolveParams conv_params = get_conv_params(ref, 0, plane);
 
 #if CONFIG_HIGHBITDEPTH
   DECLARE_ALIGNED(16, uint8_t, tmp_dst_[2 * MAX_SB_SQUARE]);
@@ -739,7 +730,7 @@
                          ? CONVERT_TO_BYTEPTR(tmp_dst_)
                          : tmp_dst_;
   av1_make_inter_predictor(pre, pre_stride, tmp_dst, MAX_SB_SIZE, subpel_x,
-                           subpel_y, sf, w, h, &conv_params, tmp_ipf,
+                           subpel_y, sf, w, h, &conv_params, interp_filter,
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                            warp_types, p_col, p_row, plane, ref,
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
@@ -782,7 +773,7 @@
 #else  // CONFIG_HIGHBITDEPTH
   DECLARE_ALIGNED(16, uint8_t, tmp_dst[MAX_SB_SQUARE]);
   av1_make_inter_predictor(pre, pre_stride, tmp_dst, MAX_SB_SIZE, subpel_x,
-                           subpel_y, sf, w, h, &conv_params, tmp_ipf,
+                           subpel_y, sf, w, h, &conv_params, interp_filter,
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
                            warp_types, p_col, p_row, plane, ref,
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
@@ -834,7 +825,7 @@
   MV32 mv = av1_scale_mv(&mv_q4, x, y, sf);
   const int subpel_x = mv.col & SUBPEL_MASK;
   const int subpel_y = mv.row & SUBPEL_MASK;
-  ConvolveParams conv_params = get_conv_params(ref, plane);
+  ConvolveParams conv_params = get_conv_params(ref, ref, plane);
 
   src += (mv.row >> SUBPEL_BITS) * src_stride + (mv.col >> SUBPEL_BITS);
 
@@ -1011,7 +1002,7 @@
           MV32 scaled_mv;
           int xs, ys, subpel_x, subpel_y;
           const int is_scaled = av1_is_scaled(sf);
-          ConvolveParams conv_params = get_conv_params(ref, plane);
+          ConvolveParams conv_params = get_conv_params(ref, ref, plane);
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
           WarpTypesAllowed warp_types;
 #if CONFIG_GLOBAL_MOTION
@@ -1151,9 +1142,9 @@
 
 #if CONFIG_CONVOLVE_ROUND
     ConvolveParams conv_params =
-        get_conv_params_no_round(ref, plane, tmp_dst, MAX_SB_SIZE);
+        get_conv_params_no_round(ref, ref, plane, tmp_dst, MAX_SB_SIZE);
 #else
-    ConvolveParams conv_params = get_conv_params(ref, plane);
+    ConvolveParams conv_params = get_conv_params(ref, ref, plane);
 #endif  // CONFIG_CONVOLVE_ROUND
     for (ref = 0; ref < 1 + is_compound; ++ref) {
 #if CONFIG_INTRABC
@@ -1174,6 +1165,7 @@
 #endif  // CONFIG_WARPED_MOTION
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
       conv_params.ref = ref;
+      conv_params.do_average = ref;
 #if CONFIG_EXT_INTER
       if (ref && is_masked_compound_type(mi->mbmi.interinter_compound_type))
         av1_make_masked_inter_predictor(
@@ -1253,7 +1245,7 @@
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
 
   for (ref = 0; ref < 1 + is_compound; ++ref) {
-    ConvolveParams conv_params = get_conv_params(ref, plane);
+    ConvolveParams conv_params = get_conv_params(ref, ref, plane);
     const uint8_t *pre =
         &pd->pre[ref].buf[(ir * pd->pre[ref].stride + ic) << 2];
 #if CONFIG_GLOBAL_MOTION
@@ -2930,7 +2922,7 @@
   MV32 scaled_mv;
   int xs, ys, subpel_x, subpel_y;
   const int is_scaled = av1_is_scaled(sf);
-  ConvolveParams conv_params = get_conv_params(0, plane);
+  ConvolveParams conv_params = get_conv_params(ref, 0, plane);
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
   WarpTypesAllowed warp_types;
 #if CONFIG_GLOBAL_MOTION
diff --git a/av1/common/reconinter.h b/av1/common/reconinter.h
index 887651f..dfb1e7d 100644
--- a/av1/common/reconinter.h
+++ b/av1/common/reconinter.h
@@ -59,6 +59,7 @@
   const InterpFilterParams interp_filter_params_y = interp_filter_params_x;
 #endif
 
+  assert(conv_params->do_average == 0 || conv_params->do_average == 1);
   assert(sf);
   if (has_scale(xs, ys)) {
     av1_convolve_c(src, src_stride, dst, dst_stride, w, h, interp_filter,
@@ -87,7 +88,7 @@
           av1_get_interp_filter_subpel_kernel(interp_filter_params_x, subpel_x);
       const int16_t *kernel_y =
           av1_get_interp_filter_subpel_kernel(interp_filter_params_y, subpel_y);
-      sf->predict[subpel_x != 0][subpel_y != 0][conv_params->ref](
+      sf->predict[subpel_x != 0][subpel_y != 0][conv_params->do_average](
           src, src_stride, dst, dst_stride, kernel_x, xs, kernel_y, ys, w, h);
     } else {
       av1_convolve(src, src_stride, dst, dst_stride, w, h, interp_filter,
@@ -110,10 +111,8 @@
 #endif
                                           int xs, int ys, int bd) {
   const int ref = conv_params->ref;
-  // ref > 0 means this is the second reference frame
-  // first reference frame's prediction result is already in dst
-  // therefore we need to average the first and second results
-  const int avg = ref > 0;
+  const int avg = conv_params->do_average;
+  assert(avg == 0 || avg == 1);
 #if CONFIG_DUAL_FILTER
   const InterpFilterParams interp_filter_params_x =
       av1_get_interp_filter_params(interp_filter[1 + 2 * ref]);
@@ -148,7 +147,7 @@
           av1_get_interp_filter_subpel_kernel(interp_filter_params_x, subpel_x);
       const int16_t *kernel_y =
           av1_get_interp_filter_subpel_kernel(interp_filter_params_y, subpel_y);
-      sf->highbd_predict[subpel_x != 0][subpel_y != 0][ref](
+      sf->highbd_predict[subpel_x != 0][subpel_y != 0][avg](
           src, src_stride, dst, dst_stride, kernel_x, xs, kernel_y, ys, w, h,
           bd);
     } else {
@@ -419,19 +418,14 @@
   if (do_warp) {
     const struct macroblockd_plane *const pd = &xd->plane[plane];
     const struct buf_2d *const pre_buf = &pd->pre[ref];
-#if CONFIG_EXT_INTER
-    int compute_avg =
-        ref && mi->mbmi.interinter_compound_type == COMPOUND_AVERAGE;
-#else
-    int compute_avg = ref;
-#endif  // CONFIG_EXT_INTER
     av1_warp_plane(&final_warp_params,
 #if CONFIG_HIGHBITDEPTH
                    xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH, xd->bd,
 #endif  // CONFIG_HIGHBITDEPTH
                    pre_buf->buf0, pre_buf->width, pre_buf->height,
                    pre_buf->stride, dst, p_col, p_row, w, h, dst_stride,
-                   pd->subsampling_x, pd->subsampling_y, xs, ys, compute_avg);
+                   pd->subsampling_x, pd->subsampling_y, xs, ys,
+                   conv_params->do_average);
     return;
   }
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
diff --git a/av1/common/x86/av1_convolve_ssse3.c b/av1/common/x86/av1_convolve_ssse3.c
index 5e627eb..e85c15e 100644
--- a/av1/common/x86/av1_convolve_ssse3.c
+++ b/av1/common/x86/av1_convolve_ssse3.c
@@ -676,11 +676,12 @@
   __m128i verf[6];
   __m128i horf[2];
   SubpelFilterCoeffs hCoeffs, vCoeffs;
+  assert(conv_params->do_average == 0 || conv_params->do_average == 1);
   const uint8_t *src_ptr;
-  store_pixel_t store2p = store2pixelTab[conv_params->ref];
-  store_pixel_t store4p = store4pixelTab[conv_params->ref];
-  transpose_to_dst_t transpose_4x4 = trans4x4Tab[conv_params->ref];
-  transpose_to_dst_t transpose_8x8 = trans8x8Tab[conv_params->ref];
+  store_pixel_t store2p = store2pixelTab[conv_params->do_average];
+  store_pixel_t store4p = store4pixelTab[conv_params->do_average];
+  transpose_to_dst_t transpose_4x4 = trans4x4Tab[conv_params->do_average];
+  transpose_to_dst_t transpose_8x8 = trans8x8Tab[conv_params->do_average];
 
   const int tapsNum = filter_params.taps;
   int block_height, block_residu;
@@ -890,10 +891,11 @@
   __m128i verf[6];
   SubpelFilterCoeffs vCoeffs;
   const uint8_t *src_ptr;
+  assert(conv_params->do_average == 0 || conv_params->do_average == 1);
   uint8_t *dst_ptr = dst;
-  store_pixel_t store2p = store2pixelTab[conv_params->ref];
-  store_pixel_t store4p = store4pixelTab[conv_params->ref];
-  store_pixel_t store8p = store8pixelTab[conv_params->ref];
+  store_pixel_t store2p = store2pixelTab[conv_params->do_average];
+  store_pixel_t store4p = store4pixelTab[conv_params->do_average];
+  store_pixel_t store8p = store8pixelTab[conv_params->do_average];
   const int tapsNum = filter_params.taps;
 
   if (0 == subpel_y_q4 || 16 != y_step_q4) {
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 7212709..27a9403 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -5170,14 +5170,6 @@
   const int refs[2] = { mbmi->ref_frame[0], mbmi->ref_frame[1] };
   int_mv ref_mv[2];
   int ite, ref;
-#if CONFIG_DUAL_FILTER
-  InterpFilter interp_filter[4] = {
-    mbmi->interp_filter[0], mbmi->interp_filter[1], mbmi->interp_filter[2],
-    mbmi->interp_filter[3],
-  };
-#else
-  const InterpFilter interp_filter = mbmi->interp_filter;
-#endif  // CONFIG_DUAL_FILTER
   struct scale_factors sf;
   struct macroblockd_plane *const pd = &xd->plane[0];
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
@@ -5260,7 +5252,7 @@
                        // odd iterations search in the second. The predictor
                        // found for the 'other' reference frame is factored in.
     const int plane = 0;
-    ConvolveParams conv_params = get_conv_params(0, plane);
+    ConvolveParams conv_params = get_conv_params(!id, 0, plane);
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
     WarpTypesAllowed warp_types;
 #if CONFIG_GLOBAL_MOTION
@@ -5275,21 +5267,13 @@
     ref_yv12[0] = xd->plane[plane].pre[0];
     ref_yv12[1] = xd->plane[plane].pre[1];
 
-#if CONFIG_DUAL_FILTER
-    // reload the filter types
-    interp_filter[0] =
-        (id == 0) ? mbmi->interp_filter[2] : mbmi->interp_filter[0];
-    interp_filter[1] =
-        (id == 0) ? mbmi->interp_filter[3] : mbmi->interp_filter[1];
-#endif  // CONFIG_DUAL_FILTER
-
 // Get the prediction block from the 'other' reference frame.
 #if CONFIG_HIGHBITDEPTH
     if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
       second_pred = CONVERT_TO_BYTEPTR(second_pred_alloc_16);
       av1_highbd_build_inter_predictor(
           ref_yv12[!id].buf, ref_yv12[!id].stride, second_pred, pw,
-          &frame_mv[refs[!id]].as_mv, &sf, pw, ph, 0, interp_filter,
+          &frame_mv[refs[!id]].as_mv, &sf, pw, ph, 0, mbmi->interp_filter,
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
           &warp_types, p_col, p_row,
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
@@ -5297,13 +5281,14 @@
     } else {
       second_pred = (uint8_t *)second_pred_alloc_16;
 #endif  // CONFIG_HIGHBITDEPTH
-      av1_build_inter_predictor(
-          ref_yv12[!id].buf, ref_yv12[!id].stride, second_pred, pw,
-          &frame_mv[refs[!id]].as_mv, &sf, pw, ph, &conv_params, interp_filter,
+      av1_build_inter_predictor(ref_yv12[!id].buf, ref_yv12[!id].stride,
+                                second_pred, pw, &frame_mv[refs[!id]].as_mv,
+                                &sf, pw, ph, &conv_params, mbmi->interp_filter,
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
-          &warp_types, p_col, p_row, plane, !id,
+                                &warp_types, p_col, p_row, plane, !id,
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
-          MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd);
+                                MV_PRECISION_Q3, mi_col * MI_SIZE,
+                                mi_row * MI_SIZE, xd);
 #if CONFIG_HIGHBITDEPTH
     }
 #endif  // CONFIG_HIGHBITDEPTH
@@ -5889,14 +5874,6 @@
   MACROBLOCKD *xd = &x->e_mbd;
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   const int other_ref = mbmi->ref_frame[!ref_idx];
-#if CONFIG_DUAL_FILTER
-  InterpFilter interp_filter[2] = {
-    (ref_idx == 0) ? mbmi->interp_filter[2] : mbmi->interp_filter[0],
-    (ref_idx == 0) ? mbmi->interp_filter[3] : mbmi->interp_filter[1]
-  };
-#else
-  const InterpFilter interp_filter = mbmi->interp_filter;
-#endif  // CONFIG_DUAL_FILTER
   struct scale_factors sf;
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
   struct macroblockd_plane *const pd = &xd->plane[0];
@@ -5943,7 +5920,7 @@
   struct buf_2d ref_yv12;
 
   const int plane = 0;
-  ConvolveParams conv_params = get_conv_params(0, plane);
+  ConvolveParams conv_params = get_conv_params(!ref_idx, 0, plane);
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
   WarpTypesAllowed warp_types;
 #if CONFIG_GLOBAL_MOTION
@@ -5962,7 +5939,7 @@
   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
     av1_highbd_build_inter_predictor(
         ref_yv12.buf, ref_yv12.stride, second_pred, pw, other_mv, &sf, pw, ph,
-        0, interp_filter,
+        0, mbmi->interp_filter,
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
         &warp_types, p_col, p_row,
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
@@ -5971,7 +5948,7 @@
 #endif  // CONFIG_HIGHBITDEPTH
     av1_build_inter_predictor(
         ref_yv12.buf, ref_yv12.stride, second_pred, pw, other_mv, &sf, pw, ph,
-        &conv_params, interp_filter,
+        &conv_params, mbmi->interp_filter,
 #if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
         &warp_types, p_col, p_row, plane, !ref_idx,
 #endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index ba3dd19..21bb087 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -41,7 +41,7 @@
   enum mv_precision mv_precision_uv;
   int uv_stride;
   // TODO(angiebird): change plane setting accordingly
-  ConvolveParams conv_params = get_conv_params(which_mv, 0);
+  ConvolveParams conv_params = get_conv_params(which_mv, which_mv, 0);
 
 #if USE_TEMPORALFILTER_12TAP
 #if CONFIG_DUAL_FILTER