diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 9c51909..3c49a19 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -4183,7 +4183,8 @@
     // encode set of lower layer frames.
     // TODO(Remya): Move ref_frame_map from AV1_COMMON to AV1_PRIMARY to avoid
     // copy.
-    if (ppi->gf_group.frame_parallel_level[cpi->gf_frame_index - 1] == 1 &&
+    if (ppi->gf_group.frame_parallel_level[cpi->gf_frame_index] == 2 &&
+        ppi->gf_group.frame_parallel_level[cpi->gf_frame_index - 1] == 1 &&
         ppi->gf_group.update_type[cpi->gf_frame_index - 1] ==
             INTNL_ARF_UPDATE) {
       memcpy(cm->ref_frame_map, ppi->ref_frame_map_copy,