commit 0e17c8787853ba2daef37c46e511a1157479b852
author Cherma Rajan A <cherma.rajan@ittiam.com> Fri Apr 30 15:39:19 2021 +0530
committer Yunqing Wang <yunqingwang@google.com> Fri Apr 30 17:43:59 2021 +0000
tree b2402a9aa781b8bac1a1b076c746954c163ac0ff
parent 2b52ff16d79e07067188c6ef0fb1a0ca5a4bedc0

Fix integer overflow issue in pack bitstream buffer computation

This CL fixes integer overflow issue identified in computation of
buffer used in pack bitstream multithreading for 32-bit builds.

BUG=aomedia:3040

Change-Id: I5d4f4bf9c5a15232b4985f570b1e2a0e5cf19363

av1/encoder/ethread.c

diff --git a/av1/encoder/ethread.c b/av1/encoder/ethread.c
index ce5a672..6892d58 100644
--- a/av1/encoder/ethread.c
+++ b/av1/encoder/ethread.c
@@ -1839,7 +1839,8 @@
     this_chunk_size = *remain_buf_size;
     *remain_buf_size = 0;
   } else {
-    this_chunk_size = (max_buf_size * tg_or_tile_size) / frame_or_tg_size;
+    const uint64_t size_scale = (uint64_t)max_buf_size * tg_or_tile_size;
+    this_chunk_size = (size_t)(size_scale / frame_or_tg_size);
     *remain_buf_size -= this_chunk_size;
     assert(*remain_buf_size > 0);
   }