commit 3f93e4cbb9cd8c4a6a0fb51698a9ced9c01aef10
author Sarah Parker <sarahparker@google.com> Wed Dec 19 10:20:02 2018 -0800
committer Sarah Parker <sarahparker@google.com> Thu Dec 20 20:41:57 2018 +0000
tree 39afb1f3f7260d690828d76d92a117e0d4cb869a
parent ab99b48306968213bba8e5945e43c5edfff3ff82

Restore functionality to FINAL_PASS_TRELLIS_OPT

If the baseline enables the full trellis optimization, we have the following
results on 150 frames of lowres at speed 0:

NO_TRELLIS_OPT: 1.76% decrease in performance
FINAL_PASS_TRELLIS_OPT: 2.176% decrease in performance

Change-Id: I2e7cf0ceb418dc3adf04111e2529446f12841fcc

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index d38321b..39dcf20 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -226,7 +226,7 @@
   if (!is_blk_skip(x, plane, blk_row * bw + blk_col) && !mbmi->skip_mode) {
     TX_TYPE tx_type = av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col,
                                       tx_size, cm->reduced_tx_set_used);
-    if (args->enable_optimize_b) {
+    if (args->enable_optimize_b != NO_TRELLIS_OPT) {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, tx_type, AV1_XFORM_QUANT_FP);
       TXB_CTX txb_ctx;
@@ -578,7 +578,7 @@
 
     const ENTROPY_CONTEXT *a = &args->ta[blk_col];
     const ENTROPY_CONTEXT *l = &args->tl[blk_row];
-    if (args->enable_optimize_b) {
+    if (args->enable_optimize_b != NO_TRELLIS_OPT) {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, tx_type, AV1_XFORM_QUANT_FP);
       TXB_CTX txb_ctx;

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 880d018..f27a92f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3160,8 +3160,7 @@
     if (plane == 0) mbmi->txk_type[txk_type_idx] = tx_type;
     RD_STATS this_rd_stats;
     av1_invalid_rd_stats(&this_rd_stats);
-
-    if (!cpi->optimize_seg_arr[mbmi->segment_id]) {
+    if (cpi->optimize_seg_arr[mbmi->segment_id] != FULL_TRELLIS_OPT) {
       av1_xform_quant(
           cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, tx_type,
           USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
@@ -3307,7 +3306,7 @@
     // if the last search tx_type is the best tx_type, we don't need to
     // do this again
     if (best_tx_type != last_tx_type) {
-      if (!cpi->optimize_seg_arr[mbmi->segment_id]) {
+      if (cpi->optimize_seg_arr[mbmi->segment_id] != FULL_TRELLIS_OPT) {
         av1_xform_quant(
             cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
             best_tx_type,