commit 5b8c5fdc55f3e637073edfbdb53487f8d6404d82
author Ryan <ryan.z.lei@intel.com> Thu Jan 11 10:53:45 2018 -0800
committer Sarah Parker <sarahparker@google.com> Thu Jan 11 22:49:58 2018 +0000
tree cec6212ffd1f5a94db142c04fcc5c6c745745e28
parent 51fac92ac5fc3877a716f53e6d5399330c5405bb

fix bug to use b_quantizer when trellis quantization is disabled.

when trellis quantizaiton is disabled, the b_quantizer should be used.
existing code has a lot of hard coded flag not hooked up correctly with
high level trellis quantization enable/disable flag, which causes
fp_quantizer is used. this patch is intended to remove the hard coded
flags and correctly get them from the high level control.

Change-Id: I0142aace03586d07aedea138fce08d3e72d99c32

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 93f1fd3..2028458 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4467,7 +4467,8 @@
 #endif  // CONFIG_CFL
     mbmi->skip = 1;
     for (int plane = 0; plane < num_planes; ++plane) {
-      av1_encode_intra_block_plane(cpi, x, bsize, plane, 1, mi_row, mi_col);
+      av1_encode_intra_block_plane(cpi, x, bsize, plane, x->optimize, mi_row,
+                                   mi_col);
     }
 #if CONFIG_CFL
     xd->cfl.store_y = 0;

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index fe1d41d..173320f 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -749,7 +749,8 @@
   MACROBLOCKD *const xd = &x->e_mbd;
   struct optimize_ctx ctx;
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
-  struct encode_b_args arg = { cpi, x, &ctx, &mbmi->skip, NULL, NULL, 1 };
+  struct encode_b_args arg = { cpi,  x,    &ctx,       &mbmi->skip,
+                               NULL, NULL, x->optimize };
   int plane;
 
   mbmi->skip = 1;

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index d40053a..e03fe7f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1893,7 +1893,8 @@
 
 #if !CONFIG_TXK_SEL
   // full forward transform and quantization
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT) {
+  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
+      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_B);
   } else {
@@ -3537,7 +3538,8 @@
                       NULL, 0, bw, bh);
   }
 
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT) {
+  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
+      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_B);
 
@@ -8702,8 +8704,8 @@
       // during luma RDO, so we can store reconstructed luma values
       memcpy(x->blk_skip[0], ctx->blk_skip[0],
              sizeof(uint8_t) * ctx->num_4x4_blk);
-      av1_encode_intra_block_plane(cpi, x, bsize, AOM_PLANE_Y, 1, mi_row,
-                                   mi_col);
+      av1_encode_intra_block_plane(cpi, x, bsize, AOM_PLANE_Y, x->optimize,
+                                   mi_row, mi_col);
       xd->cfl.store_y = 0;
     }
 #endif  // CONFIG_CFL