Fix bug in av1_optimize_b when tx64x64 is enabled

In both versions of av1_optimize_b, the code to calculate the
dequantized coefficient from a modified quantized coefficient
was not compatible with tx64x64.

Specifically, the new dequantized coefficient was calculated as an
offset applied to the original coefficient. This had a fixup block
for 32x32 transforms, but not for 64x64 transforms.

For simplicity, instead of extending the fixup block, we now
recalculate the dequantized coefficient directly.

BUG=aomedia:537

Change-Id: I86c1c8820f324ad3f6afd6a720ff59f754384a19

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 15012a9..b4d19ff 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -175,8 +175,6 @@
 #if CONFIG_NEW_QUANT
   int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
   const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq];
-#elif !CONFIG_AOM_QM
-  const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift };
 #endif  // CONFIG_NEW_QUANT
   int sz = 0;
   const int64_t rddiv = mb->rddiv;
@@ -388,22 +386,10 @@
           dqc_a = shift ? ROUND_POWER_OF_TWO(dqc_a, shift) : dqc_a;
           if (sz) dqc_a = -dqc_a;
 #else
-// The 32x32 transform coefficient uses half quantization step size.
-// Account for the rounding difference in the dequantized coefficeint
-// value when the quantization index is dropped from an even number
-// to an odd number.
-
-#if CONFIG_AOM_QM
-          tran_low_t offset = dqv >> shift;
-#else
-          tran_low_t offset = dq_step[rc != 0];
-#endif
-          if (shift & x_a) offset += (dqv & 0x01);
-
-          if (sz == 0)
-            dqc_a = dqcoeff[rc] - offset;
+          if (x_a < 0)
+            dqc_a = -((-x_a * dqv) >> shift);
           else
-            dqc_a = dqcoeff[rc] + offset;
+            dqc_a = (x_a * dqv) >> shift;
 #endif  // CONFIG_NEW_QUANT
         } else {
           dqc_a = 0;
@@ -513,8 +499,6 @@
 #if CONFIG_NEW_QUANT
   int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
   const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq];
-#elif !CONFIG_AOM_QM
-  const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift };
 #endif  // CONFIG_NEW_QUANT
   int next = eob, sz = 0;
   const int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][plane_type]) >> 1;
@@ -749,22 +733,10 @@
                                  : tokens[i][1].dqc;
         if (sz) tokens[i][1].dqc = -tokens[i][1].dqc;
 #else
-// The 32x32 transform coefficient uses half quantization step size.
-// Account for the rounding difference in the dequantized coefficeint
-// value when the quantization index is dropped from an even number
-// to an odd number.
-
-#if CONFIG_AOM_QM
-        tran_low_t offset = dqv >> shift;
-#else
-        tran_low_t offset = dq_step[rc != 0];
-#endif
-        if (shift & x) offset += (dqv & 0x01);
-
-        if (sz == 0)
-          tokens[i][1].dqc = dqcoeff[rc] - offset;
+        if (x < 0)
+          tokens[i][1].dqc = -((-x * dqv) >> shift);
         else
-          tokens[i][1].dqc = dqcoeff[rc] + offset;
+          tokens[i][1].dqc = (x * dqv) >> shift;
 #endif  // CONFIG_NEW_QUANT
       } else {
         tokens[i][1].dqc = 0;