commit 5b8c5fdc55f3e637073edfbdb53487f8d6404d82
author Ryan <ryan.z.lei@intel.com> Thu Jan 11 10:53:45 2018 -0800
committer Sarah Parker <sarahparker@google.com> Thu Jan 11 22:49:58 2018 +0000
tree cec6212ffd1f5a94db142c04fcc5c6c745745e28
parent 51fac92ac5fc3877a716f53e6d5399330c5405bb

fix bug to use b_quantizer when trellis quantization is disabled.

when trellis quantizaiton is disabled, the b_quantizer should be used.
existing code has a lot of hard coded flag not hooked up correctly with
high level trellis quantization enable/disable flag, which causes
fp_quantizer is used. this patch is intended to remove the hard coded
flags and correctly get them from the high level control.

Change-Id: I0142aace03586d07aedea138fce08d3e72d99c32

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index d40053a..e03fe7f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1893,7 +1893,8 @@
 
 #if !CONFIG_TXK_SEL
   // full forward transform and quantization
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT) {
+  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
+      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_B);
   } else {
@@ -3537,7 +3538,8 @@
                       NULL, 0, bw, bh);
   }
 
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT) {
+  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
+      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_B);
 
@@ -8702,8 +8704,8 @@
       // during luma RDO, so we can store reconstructed luma values
       memcpy(x->blk_skip[0], ctx->blk_skip[0],
              sizeof(uint8_t) * ctx->num_4x4_blk);
-      av1_encode_intra_block_plane(cpi, x, bsize, AOM_PLANE_Y, 1, mi_row,
-                                   mi_col);
+      av1_encode_intra_block_plane(cpi, x, bsize, AOM_PLANE_Y, x->optimize,
+                                   mi_row, mi_col);
       xd->cfl.store_y = 0;
     }
 #endif  // CONFIG_CFL