commit 53d45d971df8e754b0ac4b6b10d0c255213aad05
author Debargha Mukherjee <debargha@google.com> Mon Feb 11 12:59:50 2019 -0800
committer Debargha Mukherjee <debargha@google.com> Mon Feb 11 22:29:10 2019 +0000
tree a62a7649f077547d83b37d7a33d3b987e6cc141e
parent e1de4c8474ecdc101d8f99ee5534c4bf8ca2b6b5

Prevent an integer overflow

Shows up if trellis is turned on for 1-pass encodes.

BUG=aomedia:2322

Change-Id: I75df5580873d5ecd51f0fafcff4c7ced5c9e9e3b

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 0f1a266..1cfd50b 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -1650,9 +1650,7 @@
       &x->eob_costs[eob_multi_size][plane_type];
 
   const int shift = av1_get_tx_scale(tx_size);
-  const int64_t rdmult =
-      ((x->rdmult * plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8))) +
-       2) >>
+  const int rshift =
       (sharpness +
        (cpi->oxcf.aq_mode == VARIANCE_AQ && mbmi->segment_id < 4
             ? 7 - mbmi->segment_id
@@ -1661,6 +1659,11 @@
                 cpi->oxcf.deltaq_mode > NO_DELTA_Q && x->sb_energy_level < 0
             ? (3 - x->sb_energy_level)
             : 0));
+  const int64_t rdmult =
+      (((int64_t)x->rdmult *
+        (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) +
+       2) >>
+      rshift;
 
   uint8_t levels_buf[TX_PAD_2D];
   uint8_t *const levels = set_levels(levels_buf, width);
@@ -1798,7 +1801,8 @@
 
   const int shift = av1_get_tx_scale(tx_size);
   const int64_t rdmult =
-      ((x->rdmult * plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8))) +
+      (((int64_t)x->rdmult * plane_rd_mult[is_inter][plane_type]
+        << (2 * (xd->bd - 8))) +
        2) >>
       2;
   uint8_t levels_buf[TX_PAD_2D];