limit single memory allocation to be less than 8 GB

Observed max single allocation of ~500 M when encoding 1080p.
8 G should be good for up to 8K input.

BUG=aomedia:854

Change-Id: Ibf21ecbafb252410d12a8e167c603e98baaa11bc

aom_mem/aom_mem.c

diff --git a/aom_mem/aom_mem.c b/aom_mem/aom_mem.c
index 66a0c08..93f7f80 100644
--- a/aom_mem/aom_mem.c
+++ b/aom_mem/aom_mem.c
@@ -18,6 +18,21 @@
 #include "include/aom_mem_intrnl.h"
 #include "aom/aom_integer.h"
 
+#ifndef AOM_MAX_ALLOCABLE_MEMORY
+#define AOM_MAX_ALLOCABLE_MEMORY 8589934592  // 8 GB
+#endif
+
+#if defined(AOM_MAX_ALLOCABLE_MEMORY)
+// Returns 0 in case of overflow of nmemb * size.
+static int check_size_argument_overflow(uint64_t nmemb, uint64_t size) {
+  const uint64_t total_size = nmemb * size;
+  if (nmemb == 0) return 1;
+  if (size > AOM_MAX_ALLOCABLE_MEMORY / nmemb) return 0;
+  if (total_size != (size_t)total_size) return 0;
+  return 1;
+}
+#endif
+
 static size_t GetAlignedMallocSize(size_t size, size_t align) {
   return size + align - 1 + ADDRESS_STORAGE_SIZE;
 }
@@ -40,6 +55,9 @@
 void *aom_memalign(size_t align, size_t size) {
   void *x = NULL;
   const size_t aligned_size = GetAlignedMallocSize(size, align);
+#if defined(AOM_MAX_ALLOCABLE_MEMORY)
+  if (!check_size_argument_overflow(1, aligned_size)) return NULL;
+#endif
   void *const addr = malloc(aligned_size);
   if (addr) {
     x = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE, align);