commit acaa384f23ed85b0b4f78815e58363c75613c2fe
author Yunqing Wang <yunqingwang@google.com> Fri Apr 21 09:33:33 2017 -0700
committer Yunqing Wang <yunqingwang@google.com> Mon Apr 24 16:35:52 2017 +0000
tree ca0ebd85f6f481797936f2bc8cc3894559369f6b
parent 89912f9a331bf0f5e1b4ca8f563ba02964dcfad4

Port an integer overflow fix in mcomp.c

Ported a bug fix from VP9.
(https://chromium-review.googlesource.com/c/473566/)

Change-Id: I5669c3878b34a5fda4ce1aabb8c653ea823d5e42

av1/encoder/mcomp.c

diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index c77a43c..e7d8c58 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -81,10 +81,8 @@
                        int *mvcost[2], int error_per_bit) {
   if (mvcost) {
     const MV diff = { mv->row - ref->row, mv->col - ref->col };
-    // This product sits at a 32-bit ceiling right now and any additional
-    // accuracy in either bit cost or error cost will cause it to overflow.
-    return ROUND_POWER_OF_TWO(
-        (unsigned)mv_cost(&diff, mvjcost, mvcost) * error_per_bit,
+    return (int)ROUND_POWER_OF_TWO_64(
+        (int64_t)mv_cost(&diff, mvjcost, mvcost) * error_per_bit,
         RDDIV_BITS + AV1_PROB_COST_SHIFT - RD_EPB_SHIFT +
             PIXEL_TRANSFORM_ERROR_SCALE);
   }