commit 41d286f3746bdb17bbe4ad0f0d45504af514ffde
author Wan-Teh Chang <wtc@google.com> Wed Oct 03 11:43:03 2018 -0700
committer Wan-Teh Chang <wtc@google.com> Wed Oct 03 20:42:41 2018 +0000
tree 5502328c20ad41dfdbebf17c8a9a020a65a80607
parent 0c1b8c1f02ebc26e6011926ae43b74e82ed97abc

Release reference to the new frame buffer on error

When we get a new frame buffer from get_free_fb(), we may temporarily
save that reference in a local variable. In that case, if the
aom_realloc_frame_buffer() call fails, we need to release that reference
before calling aom_internal_error(), otherwise the reference saved in
the local variable will be leaked.

BUG=oss-fuzz:10780

Change-Id: I8f2727a6b9e09eca8b819561305b9670ebac455e

av1/decoder/decodeframe.c

diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 9cb1cee..285e365 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -5067,6 +5067,7 @@
                   AOM_BORDER_IN_PIXELS, cm->byte_alignment,
                   &pool->frame_bufs[buf_idx].raw_frame_buffer, pool->get_fb_cb,
                   pool->cb_priv)) {
+            decrease_ref_count(buf_idx, frame_bufs, pool);
             unlock_buffer_pool(pool);
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Failed to allocate frame buffer");

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 43ddaf3..47f2a5e 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -3754,9 +3754,14 @@
                   &new_fb_ptr->buf, cm->width, cm->height,
                   cm->seq_params.subsampling_x, cm->seq_params.subsampling_y,
                   cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS,
-                  cm->byte_alignment, NULL, NULL, NULL))
+                  cm->byte_alignment, NULL, NULL, NULL)) {
+            if (force_scaling) {
+              // Release the reference acquired in the get_free_fb() call above.
+              --new_fb_ptr->ref_count;
+            }
             aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
                                "Failed to allocate frame buffer");
+          }
           av1_resize_and_extend_frame(
               ref, &new_fb_ptr->buf, (int)cm->seq_params.bit_depth, num_planes);
           cpi->scaled_ref_idx[ref_frame - 1] = new_fb;