diff --git a/av1/decoder/obu.c b/av1/decoder/obu.c
index 40171cf..64a1d90 100644
--- a/av1/decoder/obu.c
+++ b/av1/decoder/obu.c
@@ -510,7 +510,8 @@
       cm->error.error_code = AOM_CODEC_CORRUPT_FRAME;
       return 0;
     }
-    av1_set_reference_dec(cm, 0, 1, &pbi->ext_refs.refs[ref_idx]);
+    av1_set_reference_dec(cm, cm->remapped_ref_idx[0], 1,
+                          &pbi->ext_refs.refs[ref_idx]);
 
     pbi->dec_tile_row = aom_rb_read_literal(rb, 8);
     pbi->dec_tile_col = aom_rb_read_literal(rb, 8);