diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index a1dd155..96ae8b4 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1099,8 +1099,4 @@
   av1_init_mv_probs(cm);
   cm->fc->initialized = 1;
   av1_setup_frame_contexts(cm);
-
-  // prev_mi will only be allocated in encoder.
-  if (frame_is_intra_only(cm) && cm->prev_mi)
-    memset(cm->prev_mi, 0, cm->mi_stride * cm->mi_rows * sizeof(*cm->prev_mi));
 }
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index dde3c12..9347e0c 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -430,10 +430,6 @@
   int mi_alloc_size;
   MB_MODE_INFO *mi;  /* Corresponds to upper left visible macroblock */
 
-  // TODO(agrange): Move prev_mi into encoder structure.
-  // prev_mi will only be allocated in encoder.
-  MB_MODE_INFO *prev_mi;  /* 'mi' from last frame */
-
   // Separate mi functions between encoder and decoder.
   int (*alloc_mi)(struct AV1Common *cm, int mi_size);
   void (*free_mi)(struct AV1Common *cm);
@@ -442,7 +438,6 @@
   // Grid of pointers to 4x4 MB_MODE_INFO structs. Any 4x4 not in the visible
   // area will be NULL.
   MB_MODE_INFO **mi_grid_base;
-  MB_MODE_INFO **prev_mi_grid_base;
 
   // Whether to use previous frames' motion vectors for prediction.
   int allow_ref_frame_mvs;
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index a3bc309..a8cc1f8 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -378,14 +378,8 @@
 }
 
 static void enc_setup_mi(AV1_COMMON *cm) {
-  int i;
   int mi_rows_sb_aligned = calc_mi_size(cm->mi_rows);
   memset(cm->mi, 0, cm->mi_stride * mi_rows_sb_aligned * sizeof(*cm->mi));
-  // Clear top border row
-  memset(cm->prev_mi, 0, sizeof(*cm->prev_mi) * cm->mi_stride);
-  // Clear left border column
-  for (i = 0; i < mi_rows_sb_aligned; ++i)
-    memset(&cm->prev_mi[i * cm->mi_stride], 0, sizeof(*cm->prev_mi));
 
   memset(cm->mi_grid_base, 0,
          cm->mi_stride * mi_rows_sb_aligned * sizeof(*cm->mi_grid_base));
@@ -394,16 +388,11 @@
 static int enc_alloc_mi(AV1_COMMON *cm, int mi_size) {
   cm->mi = aom_calloc(mi_size, sizeof(*cm->mi));
   if (!cm->mi) return 1;
-  cm->prev_mi = aom_calloc(mi_size, sizeof(*cm->prev_mi));
-  if (!cm->prev_mi) return 1;
   cm->mi_alloc_size = mi_size;
 
   cm->mi_grid_base =
       (MB_MODE_INFO **)aom_calloc(mi_size, sizeof(MB_MODE_INFO *));
   if (!cm->mi_grid_base) return 1;
-  cm->prev_mi_grid_base =
-      (MB_MODE_INFO **)aom_calloc(mi_size, sizeof(MB_MODE_INFO *));
-  if (!cm->prev_mi_grid_base) return 1;
 
   return 0;
 }
@@ -411,27 +400,11 @@
 static void enc_free_mi(AV1_COMMON *cm) {
   aom_free(cm->mi);
   cm->mi = NULL;
-  aom_free(cm->prev_mi);
-  cm->prev_mi = NULL;
   aom_free(cm->mi_grid_base);
   cm->mi_grid_base = NULL;
-  aom_free(cm->prev_mi_grid_base);
-  cm->prev_mi_grid_base = NULL;
   cm->mi_alloc_size = 0;
 }
 
-static void swap_mi_and_prev_mi(AV1_COMMON *cm) {
-  // Current mi will be the prev_mi for the next frame.
-  MB_MODE_INFO **temp_base = cm->prev_mi_grid_base;
-  MB_MODE_INFO *temp = cm->prev_mi;
-  cm->prev_mi = cm->mi;
-  cm->mi = temp;
-
-  // Update the upper left visible macroblock ptrs.
-  cm->prev_mi_grid_base = cm->mi_grid_base;
-  cm->mi_grid_base = temp_base;
-}
-
 void av1_initialize_enc(void) {
   av1_rtcd();
   aom_dsp_rtcd();
@@ -5698,13 +5671,8 @@
   // it is not shown, we still need update the count down.
 
   if (cm->show_frame) {
-    // TODO(zoeliu): We may only swamp mi and prev_mi for those frames that
-    // are
-    // being used as reference.
-    swap_mi_and_prev_mi(cm);
     // Don't increment frame counters if this was an altref buffer
     // update not a real frame
-
     ++current_frame->frame_number;
   }
 
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index 68447e4..f3f554f 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -12,6 +12,7 @@
 #include <math.h>
 #include <limits.h>
 
+#include "av1/common/blockd.h"
 #include "config/aom_config.h"
 
 #include "av1/common/alloccommon.h"
@@ -1029,6 +1030,12 @@
 
   for (i = 0; i < num_planes; i++) input_buffer[i] = mbd->plane[i].pre[0].buf;
 
+  // Make a temporary mbmi for temporal filtering
+  MB_MODE_INFO **backup_mi_grid = mbd->mi;
+  MB_MODE_INFO mbmi = { 0 };
+  MB_MODE_INFO *mbmi_ptr = &mbmi;
+  mbd->mi = &mbmi_ptr;
+
   for (mb_row = 0; mb_row < mb_rows; mb_row++) {
     // Source frames are extended to 16 pixels. This is different than
     //  L/A/G reference frames that have a border of 32 (AV1ENCBORDERINPIXELS)
@@ -1305,6 +1312,8 @@
 
   // Restore input state
   for (i = 0; i < num_planes; i++) mbd->plane[i].pre[0].buf = input_buffer[i];
+
+  mbd->mi = backup_mi_grid;
 }
 
 // This is an adaptation of the mehtod in the following paper:
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index 4c3f5be..03c4849 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -493,8 +493,12 @@
   for (idx = 0; idx < INTER_REFS_PER_FRAME; ++idx)
     ref_frame[idx] = cpi->tpl_frame[tpl_frame->ref_map_index[idx]].gf_picture;
 
-  xd->mi = cm->mi_grid_base;
-  xd->mi[0] = cm->mi;
+  // Make a temporary mbmi for tpl model
+  MB_MODE_INFO **backup_mi_grid = xd->mi;
+  MB_MODE_INFO mbmi = { 0 };
+  MB_MODE_INFO *mbmi_ptr = &mbmi;
+  xd->mi = &mbmi_ptr;
+
   xd->block_ref_scale_factors[0] = &sf;
 
   const int base_qindex = gf_group->q_val[frame_idx];
@@ -538,6 +542,9 @@
                          mi_col, bsize);
     }
   }
+
+  // Restore the mi_grid
+  xd->mi = backup_mi_grid;
 }
 
 static void init_gop_frames_for_tpl(