Bound the total allocated memory of frame buffers.

By setting a debugger breakpoint in aom_memalign() (called by both
aom_malloc() and aom_calloc()) for 'size' greater than 10 MB, I found
that the frame buffer pool is one of the two biggest consumers of heap
memory in the decoder. This CL tries to bound the memory consumption of
the frame buffer pool. We could define a new macro for the maximum frame
buffer pool size, but for now I am borrowing the existing
AOM_MAX_ALLOCABLE_MEMORY macro for this purpose.

Tested:
cmake ../aom -DCONFIG_PIC=1 -DCONFIG_LOWBITDEPTH=1 -DCONFIG_SIZE_LIMIT=1 -DDECODE_HEIGHT_LIMIT=12288 -DDECODE_WIDTH_LIMIT=12288 -DAOM_EXTRA_C_FLAGS="-DAOM_MAX_ALLOCABLE_MEMORY=536870912 -DDO_RANGE_CHECK_CLAMP=1"
make
./test_libaom --gtest_filter=*TestVectorTest*

BUG=oss-fuzz:9678
BUG=oss-fuzz:9697
BUG=oss-fuzz:9956
BUG=oss-fuzz:9992
BUG=oss-fuzz:9993

Change-Id: Iab894920bb5e85aa13d92598fd18e88c84ae64fd

aom_scale/generic/yv12config.c

diff --git a/aom_scale/generic/yv12config.c b/aom_scale/generic/yv12config.c
index ca448db..5cfba88 100644
--- a/aom_scale/generic/yv12config.c
+++ b/aom_scale/generic/yv12config.c
@@ -14,6 +14,7 @@
 #include "aom_mem/aom_mem.h"
 #include "aom_ports/mem.h"
 #include "aom_scale/yv12config.h"
+#include "av1/common/enums.h"
 
 /****************************************************************************
  *  Exports
@@ -74,6 +75,17 @@
 
     uint8_t *buf = NULL;
 
+#if defined AOM_MAX_ALLOCABLE_MEMORY
+    // The size of ybf->buffer_alloc.
+    uint64_t alloc_size = frame_size;
+    // The size of ybf->y_buffer_8bit.
+    if (use_highbitdepth) alloc_size += yplane_size;
+    // The decoder may allocate REF_FRAMES frame buffers in the frame buffer
+    // pool. Bound the total amount of allocated memory as if these REF_FRAMES
+    // frame buffers were allocated in a single allocation.
+    if (alloc_size > AOM_MAX_ALLOCABLE_MEMORY / REF_FRAMES) return -1;
+#endif
+
     if (cb != NULL) {
       const int align_addr_extra_size = 31;
       const uint64_t external_frame_size = frame_size + align_addr_extra_size;