diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 9061cb8..d0a2e9d 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -3164,8 +3164,10 @@
         break;
       }
     }
-    assert(updated_fb >= 0);
-    cm->fb_of_context_type[cm->frame_context_idx] = updated_fb;
+    // large scale tile sometimes won't refresh any fbs
+    if (updated_fb >= 0) {
+      cm->fb_of_context_type[cm->frame_context_idx] = updated_fb;
+    }
 #endif
 
     if (!cpi->refresh_frame_mask) {