Add option to use fp quant when new-quant is used

Change-Id: I861ccc09454606ce4172db02a52550f9b7a7800c

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 4016628..df5a936 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -31,6 +31,12 @@
 extern "C" {
 #endif
 
+#if CONFIG_NEW_QUANT
+#define USE_B_QUANT_NO_TRELLIS 0
+#else
+#define USE_B_QUANT_NO_TRELLIS 1
+#endif  // CONFIG_NEW_QUANT
+
 #define MAX_MB_PLANE 3
 
 // Set COMPOUND_SEGMENT_TYPE to one of the three

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index ce8d1d7..bc0605e 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -585,8 +585,9 @@
       av1_optimize_b(args->cpi, x, plane, blk_row, blk_col, block, plane_bsize,
                      tx_size, a, l, CONFIG_LV_MAP);
     } else {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_B);
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
     }
   } else {
     p->eobs[block] = 0;
@@ -901,8 +902,9 @@
     }
 #endif  // CONFIG_TXK_SEL
   } else {
-    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                    AV1_XFORM_QUANT_B);
+    av1_xform_quant(
+        cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+        USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
   }
 
   av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst,

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 1df90bc..5cc611d 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2337,8 +2337,9 @@
     RD_STATS this_rd_stats;
     av1_invalid_rd_stats(&this_rd_stats);
     if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_B);
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
     } else {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, AV1_XFORM_QUANT_FP);
@@ -2376,8 +2377,9 @@
     // intra mode needs decoded result such that the next transform block
     // can use it for prediction.
     if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_B);
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
     } else {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, AV1_XFORM_QUANT_FP);

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index fd274d2..39495cf 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1858,10 +1858,10 @@
 
 #if !CONFIG_TXK_SEL
   // full forward transform and quantization
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
-      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
-    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                    AV1_XFORM_QUANT_B);
+  if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+    av1_xform_quant(
+        cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+        USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
   } else {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_FP);
@@ -3510,10 +3510,10 @@
                       NULL, 0, bw, bh);
   }
 
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
-      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
-    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                    AV1_XFORM_QUANT_B);
+  if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+    av1_xform_quant(
+        cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+        USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
 
   } else {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,