commit 32d26bc0af45681fe93c3c9a6f39078569cf707f
author Jingning Han <jingning@google.com> Sun Jun 11 07:50:39 2017 -0700
committer Jingning Han <jingning@google.com> Sun Jun 11 07:50:39 2017 -0700
tree efb784b852fe3f7c8a23b7d3cbaade69999e315d
parent 3599435aed39900f44a295685c027279067a9da1

Avoid right shift negative numbers

This avoids the rounding errors due to the right shift of the
negative numbers that cause the reconstruction coefficient has
higher distortion than the source coefficient.

BUG=aomedia:599

Change-Id: I11ed86bf1d41164dda4398545334a7b4e8e10513

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 246b656..b4624a4 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -246,7 +246,7 @@
        */
       // compute the distortion for the first candidate
       // and the distortion for quantizing to 0.
-      int dx0 = (-coeff[rc]) * (1 << shift);
+      int dx0 = abs(coeff[rc]) * (1 << shift);
 #if CONFIG_HIGHBITDEPTH
       if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
         dx0 >>= xd->bd - 8;
@@ -270,7 +270,9 @@
       dx = (dqcoeff[rc] - coeff[rc]) * (1 << shift);
 #if CONFIG_HIGHBITDEPTH
       if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
-        dx >>= xd->bd - 8;
+        int dx_sign = dx < 0 ? 1 : 0;
+        dx = abs(dx) >> (xd->bd - 8);
+        if (dx_sign) dx = -dx;
       }
 #endif  // CONFIG_HIGHBITDEPTH
       d2 = (int64_t)dx * dx;