diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index 5798a3f..032aa3a 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -597,6 +597,9 @@
 
     frame_bufs[i].ref_count = 1;
   } else {
+    // We should never run out of free buffers. If this assertion fails, there
+    // is a reference leak.
+    assert(0 && "Ran out of free frame buffers. Likely a reference leak.");
     // Reset i to be INVALID_IDX to indicate no free buffer found.
     i = INVALID_IDX;
   }
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index e098041..2c8389d 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -5061,8 +5061,6 @@
           // pixels set to neutral grey.
           buf_idx = get_free_fb(cm);
           if (buf_idx == INVALID_IDX) {
-            assert(0 &&
-                   "Ran out of free frame buffers. Likely a reference leak.");
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Unable to find free frame buffer");
           }
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c
index 5c9bfd3..ba13741 100644
--- a/av1/decoder/decoder.c
+++ b/av1/decoder/decoder.c
@@ -483,7 +483,6 @@
   // Find a free frame buffer. Return error if can not find any.
   cm->new_fb_idx = get_free_fb(cm);
   if (cm->new_fb_idx == INVALID_IDX) {
-    assert(0 && "Ran out of free frame buffers. Likely a reference leak.");
     cm->error.error_code = AOM_CODEC_MEM_ERROR;
     return 1;
   }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index f8aef23..95d47fa 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -3760,10 +3760,11 @@
         int new_fb = cpi->scaled_ref_idx[ref_frame - 1];
         if (new_fb == INVALID_IDX) {
           new_fb = get_free_fb(cm);
+          if (new_fb == INVALID_IDX)
+            aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
+                               "Unable to find free frame buffer");
           force_scaling = 1;
         }
-        // TODO(wtc): Is it OK to ignore the get_free_fb() failure?
-        if (new_fb == INVALID_IDX) return;
         new_fb_ptr = &pool->frame_bufs[new_fb];
         if (force_scaling || new_fb_ptr->buf.y_crop_width != cm->width ||
             new_fb_ptr->buf.y_crop_height != cm->height) {
@@ -6529,10 +6530,7 @@
     }
     cm->new_fb_idx = get_free_fb(cm);
 
-    if (cm->new_fb_idx == INVALID_IDX) {
-      assert(0 && "Ran out of free frame buffers. Likely a reference leak.");
-      return -1;
-    }
+    if (cm->new_fb_idx == INVALID_IDX) return -1;
 
     // Clear down mmx registers
     aom_clear_system_state();
@@ -6722,10 +6720,7 @@
   }
   cm->new_fb_idx = get_free_fb(cm);
 
-  if (cm->new_fb_idx == INVALID_IDX) {
-    assert(0 && "Ran out of free frame buffers. Likely a reference leak.");
-    return -1;
-  }
+  if (cm->new_fb_idx == INVALID_IDX) return -1;
 
   // Retain the RF_LEVEL for the current newly coded frame.
   cpi->frame_rf_level[cm->new_fb_idx] =