commit 18403941645f37a5ccefd6904d416409bf0e164e
author Jingning Han <jingning@google.com> Tue Mar 20 14:35:34 2018 -0700
committer Yaowu Xu <yaowu@google.com> Wed Mar 21 00:34:41 2018 +0000
tree 14e24d7a53f5575393834f8651ca650ef0f6c6ca
parent 5d949699014c98ac7515617c18fc2120b983dbc7

Skip soft quantization when configured

If the encoder is configured to skip the soft coefficient
quantization in the rate-distortion optimization process, follow
the same configuration in the intra block rebuild stage. This
solves an encoding assertion issue:

BUG=aomedia:1610

Change-Id: I51ff0d3f9c3e7737b0c6985d82f685272ce90305

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 6200cd8..c70deae 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2049,7 +2049,7 @@
        blk_col + tx_size_wide_unit[tx_size] < mi_size_wide[plane_bsize])) {
     // intra mode needs decoded result such that the next transform block
     // can use it for prediction.
-    if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+    if (!cpi->optimize_seg_arr[mbmi->segment_id]) {
       av1_xform_quant(
           cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
           USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);