Expand aomenc flag to disable coeff optimization

Removes the existing macro and refactors.
Also trellis quant was turned off for 1-pass encoding for
no reason. Turned it back on, causing the change in stats.

STATS_CHANGED

Change-Id: I2aa6267aeecbcf916ebea7ffa132e730876d77f0

aom/aomcx.h

diff --git a/aom/aomcx.h b/aom/aomcx.h
index fc0f233..ec5455a 100644
--- a/aom/aomcx.h
+++ b/aom/aomcx.h
@@ -575,8 +575,9 @@
    *
    *                          0 = apply trellis quantization
    *                          1 = do not apply trellis quantization
+   *                          2 = disable trellis quantization partially
    *
-   *  By default, the encoder applies trellis optimization on quantized
+   *  By default, the encoder applies optimization on quantized
    *  coefficients.
    *
    */

apps/aomenc.c

diff --git a/apps/aomenc.c b/apps/aomenc.c
index 0123354..2574477 100644
--- a/apps/aomenc.c
+++ b/apps/aomenc.c
@@ -525,7 +525,7 @@
 static const arg_def_t disable_trellis_quant =
     ARG_DEF(NULL, "disable-trellis-quant", 1,
             "Disable trellis optimization of quantized coefficients (0: false ("
-            "default) 1: true)");
+            "default) 1: true  2: partial true)");
 static const arg_def_t enable_qm =
     ARG_DEF(NULL, "enable-qm", 1,
             "Enable quantisation matrices (0: false (default), 1: true)");

av1/av1_cx_iface.c

diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 98d6670..987a478 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -432,6 +432,8 @@
   RANGE_CHECK_HI(extra_cfg, chroma_subsampling_x, 1);
   RANGE_CHECK_HI(extra_cfg, chroma_subsampling_y, 1);
 
+  RANGE_CHECK_HI(extra_cfg, disable_trellis_quant, 2);
+
   return AOM_CODEC_OK;
 }
 

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 30a8672..0ec8bf3 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6384,7 +6384,7 @@
     if (xd->lossless[i]) {
       cpi->optimize_seg_arr[i] = 0;
     } else {
-      cpi->optimize_seg_arr[i] = cpi->optimize_speed_feature;
+      cpi->optimize_seg_arr[i] = cpi->sf.optimize_coefficients;
     }
   }
   cm->coded_lossless = is_coded_lossless(cm, xd);

av1/encoder/encoder.h

diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 9942280..9050a89 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -632,7 +632,6 @@
   struct lookahead_entry *alt_ref_source;
   int no_show_kf;
 
-  int optimize_speed_feature;
   int optimize_seg_arr[MAX_SEGMENTS];
 
   YV12_BUFFER_CONFIG *source;

av1/encoder/speed_features.c

diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 096312a..614f329 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -17,12 +17,6 @@
 
 #include "aom_dsp/aom_dsp_common.h"
 
-// Setting this to 1 will disable trellis optimization completely.
-// Setting this to 2 will disable trellis optimization within the
-// transform search. Trellis optimization will still be applied
-// in the final encode.
-#define DISABLE_TRELLISQ_SEARCH 0
-
 #define MAX_MESH_SPEED 5  // Max speed setting for mesh motion method
 // Max speed setting for tx domain evaluation
 #define MAX_TX_DOMAIN_EVAL_SPEED 5
@@ -652,18 +646,20 @@
   sf->mv.subpel_search_method = SUBPEL_TREE;
   sf->mv.subpel_iters_per_step = 2;
   sf->mv.subpel_force_stop = EIGHTH_PEL;
-#if DISABLE_TRELLISQ_SEARCH == 2
-  sf->optimize_coefficients = !is_lossless_requested(&cpi->oxcf)
-                                  ? FINAL_PASS_TRELLIS_OPT
-                                  : NO_TRELLIS_OPT;
-#elif DISABLE_TRELLISQ_SEARCH == 1
-  sf->optimize_coefficients = NO_TRELLIS_OPT;
-#else
-  if (is_lossless_requested(&cpi->oxcf))
+  if (cpi->oxcf.disable_trellis_quant == 2) {
+    sf->optimize_coefficients = !is_lossless_requested(&cpi->oxcf)
+                                    ? FINAL_PASS_TRELLIS_OPT
+                                    : NO_TRELLIS_OPT;
+  } else if (cpi->oxcf.disable_trellis_quant == 0) {
+    if (is_lossless_requested(&cpi->oxcf))
+      sf->optimize_coefficients = NO_TRELLIS_OPT;
+    else
+      sf->optimize_coefficients = FULL_TRELLIS_OPT;
+  } else if (cpi->oxcf.disable_trellis_quant == 1) {
     sf->optimize_coefficients = NO_TRELLIS_OPT;
-  else
-    sf->optimize_coefficients = FULL_TRELLIS_OPT;
-#endif  // DISABLE_TRELLISQ_SEARCH
+  } else {
+    assert(0 && "Invalid disable_trellis_quant value");
+  }
   sf->gm_erroradv_type = GM_ERRORADV_TR_0;
   sf->mv.reduce_first_step_size = 0;
   sf->mv.auto_mv_step_size = 0;
@@ -838,8 +834,9 @@
   // No recode for 1 pass.
   if (oxcf->pass == 0) {
     sf->recode_loop = DISALLOW_RECODE;
-    sf->optimize_coefficients = NO_TRELLIS_OPT;
   }
+  // FIXME: trellis not very efficient for quantization matrices
+  if (oxcf->using_qm) sf->optimize_coefficients = NO_TRELLIS_OPT;
 
   if (sf->mv.subpel_search_method == SUBPEL_TREE) {
     cpi->find_fractional_mv_step = av1_find_best_sub_pixel_tree;
@@ -851,12 +848,6 @@
     cpi->find_fractional_mv_step = av1_find_best_sub_pixel_tree_pruned_evenmore;
   }
 
-  cpi->optimize_speed_feature =
-      oxcf->pass != 1 ? sf->optimize_coefficients : NO_TRELLIS_OPT;
-  // FIXME: trellis not very efficient for quantisation matrices
-  if (oxcf->using_qm) cpi->optimize_speed_feature = NO_TRELLIS_OPT;
-  if (oxcf->disable_trellis_quant) cpi->optimize_speed_feature = NO_TRELLIS_OPT;
-
   x->min_partition_size = sf->default_min_partition_size;
   x->max_partition_size = sf->default_max_partition_size;