diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c
index 9e09c63..aeeba5c 100644
--- a/av1/encoder/encode_strategy.c
+++ b/av1/encoder/encode_strategy.c
@@ -933,8 +933,8 @@
         av1_temporal_filter(cpi, arf_src_index, update_type,
                             is_forward_keyframe, &show_existing_alt_ref);
     if (code_arf) {
-      aom_extend_frame_borders(&cpi->alt_ref_buffer, av1_num_planes(cm));
-      frame_input->source = &cpi->alt_ref_buffer;
+      aom_extend_frame_borders(&cpi->ppi->alt_ref_buffer, av1_num_planes(cm));
+      frame_input->source = &cpi->ppi->alt_ref_buffer;
       aom_copy_metadata_to_frame_buffer(frame_input->source,
                                         source_buffer->metadata);
     }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 4826ee8..5d60725 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -1431,6 +1431,7 @@
 
 void av1_remove_primary_compressor(AV1_PRIMARY *ppi) {
   if (!ppi) return;
+  aom_free_frame_buffer(&ppi->alt_ref_buffer);
   for (int i = 0; i < MAX_NUM_OPERATING_POINTS; ++i) {
     aom_free(ppi->level_params.level_info[i]);
   }
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 48ad327b..104f57f 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -2140,6 +2140,12 @@
    * Rate control related parameters.
    */
   PRIMARY_RATE_CONTROL p_rc;
+
+  /*!
+   * Frame buffer holding the temporally filtered source frame. It can be KEY
+   * frame or ARF frame.
+   */
+  YV12_BUFFER_CONFIG alt_ref_buffer;
 } AV1_PRIMARY;
 
 /*!
@@ -2410,12 +2416,6 @@
   RefBufferStack ref_buffer_stack;
 
   /*!
-   * Frame buffer holding the temporally filtered source frame. It can be KEY
-   * frame or ARF frame.
-   */
-  YV12_BUFFER_CONFIG alt_ref_buffer;
-
-  /*!
    * Tell if OVERLAY frame shows existing alt_ref frame.
    */
   int show_existing_alt_ref;
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h
index 30b9121..1021eb6 100644
--- a/av1/encoder/encoder_alloc.h
+++ b/av1/encoder/encoder_alloc.h
@@ -261,7 +261,6 @@
   aom_free_frame_buffer(&cpi->trial_frame_rst);
   aom_free_frame_buffer(&cpi->scaled_source);
   aom_free_frame_buffer(&cpi->scaled_last_source);
-  aom_free_frame_buffer(&cpi->alt_ref_buffer);
 
   free_token_info(token_info);
 
@@ -323,7 +322,7 @@
 
   // TODO(agrange) Check if ARF is enabled and skip allocation if not.
   if (aom_realloc_frame_buffer(
-          &cpi->alt_ref_buffer, oxcf->frm_dim_cfg.width,
+          &cpi->ppi->alt_ref_buffer, oxcf->frm_dim_cfg.width,
           oxcf->frm_dim_cfg.height, seq_params->subsampling_x,
           seq_params->subsampling_y, seq_params->use_highbitdepth,
           cpi->oxcf.border_in_pixels, cm->features.byte_alignment, NULL, NULL,
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 1b2ca45..f6d25ad 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -1040,7 +1040,7 @@
       int is_forward_keyframe = 0;
       av1_temporal_filter(cpi, arf_src_index, arf_update_type,
                           is_forward_keyframe, NULL);
-      aom_extend_frame_borders(&cpi->alt_ref_buffer,
+      aom_extend_frame_borders(&cpi->ppi->alt_ref_buffer,
                                av1_num_planes(&cpi->common));
     }
 
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index 4d53b75..b0c3b5d 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -862,13 +862,13 @@
       }
     }
     tf_normalize_filtered_frame(mbd, block_size, mb_row, mb_col, num_planes,
-                                accum, count, &cpi->alt_ref_buffer);
+                                accum, count, &cpi->ppi->alt_ref_buffer);
 
     if (check_show_existing) {
       const int y_height = mb_height >> mbd->plane[0].subsampling_y;
       const int y_width = mb_width >> mbd->plane[0].subsampling_x;
       const int source_y_stride = frame_to_filter->y_stride;
-      const int filter_y_stride = cpi->alt_ref_buffer.y_stride;
+      const int filter_y_stride = cpi->ppi->alt_ref_buffer.y_stride;
       const int source_offset =
           mb_row * y_height * source_y_stride + mb_col * y_width;
       const int filter_offset =
@@ -876,7 +876,8 @@
       unsigned int sse = 0;
       cpi->fn_ptr[block_size].vf(
           frame_to_filter->y_buffer + source_offset, source_y_stride,
-          cpi->alt_ref_buffer.y_buffer + filter_offset, filter_y_stride, &sse);
+          cpi->ppi->alt_ref_buffer.y_buffer + filter_offset, filter_y_stride,
+          &sse);
       diff->sum += sse;
       diff->sse += sse * (int64_t)sse;
     }
diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index a7d8434..138dff6 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -1305,7 +1305,7 @@
     }
     if (gop_eval && cpi->rc.frames_since_key > 0 &&
         gf_group->arf_index == gf_index)
-      tpl_frame->gf_picture = &cpi->alt_ref_buffer;
+      tpl_frame->gf_picture = &cpi->ppi->alt_ref_buffer;
 
     // 'cm->current_frame.frame_number' is the display number
     // of the current frame.
diff --git a/doc/dev_guide/av1_encoder.dox b/doc/dev_guide/av1_encoder.dox
index ea610f1..a19db95 100644
--- a/doc/dev_guide/av1_encoder.dox
+++ b/doc/dev_guide/av1_encoder.dox
@@ -108,10 +108,10 @@
     - \ref AV1_PRIMARY.lap_enabled
     - \ref AV1_PRIMARY.twopass (\ref TWO_PASS)
     - \ref AV1_PRIMARY.p_rc (\ref PRIMARY_RATE_CONTROL)
+    - \ref AV1_PRIMARY.alt_ref_buffer (\ref yv12_buffer_config)
 
 - \ref AV1_COMP
     - \ref AV1_COMP.oxcf (\ref AV1EncoderConfig)
-    - \ref AV1_COMP.alt_ref_buffer (\ref yv12_buffer_config)
     - \ref AV1_COMP.rc (\ref RATE_CONTROL)
     - \ref AV1_COMP.speed
     - \ref AV1_COMP.sf (\ref SPEED_FEATURES)
@@ -327,9 +327,11 @@
 The following are the main data structures referenced in this section
 (see also \ref architecture_enc_data_structures):
 
+- \ref AV1_PRIMARY ppi (the primary compressor instance data structure)
+    - \ref AV1_PRIMARY.alt_ref_buffer (\ref yv12_buffer_config)
+
 - \ref AV1_COMP cpi (the main compressor instance data structure)
     - \ref AV1_COMP.oxcf (\ref AV1EncoderConfig)
-    - \ref AV1_COMP.alt_ref_buffer (\ref yv12_buffer_config)
 
 - \ref AV1EncoderConfig (Encoder configuration parameters)
     - \ref AV1EncoderConfig.algo_cfg (\ref AlgoCfg)
@@ -451,7 +453,7 @@
 The main entry point for temporal filtering is \ref av1_temporal_filter().
 This function returns 1 if temporal filtering is successful, otherwise 0.
 When temporal filtering is applied, the filtered frame will be held in
-the frame buffer \ref AV1_COMP.alt_ref_buffer, which is the frame to be
+the frame buffer \ref AV1_PRIMARY.alt_ref_buffer, which is the frame to be
 encoded in the following encoding process.
 
 Almost all temporal filter related code is in av1/encoder/temporal_filter.c