Enable tx domain distortion optimization for winner mode

For speed >= 3, tx domain distortion type set conservatively
during mode evaluation and set pixel domain distortion for
final winner.
This change is not applicable for screen content coding.

            Encode Time
cpu-used     Reduction      Quality Loss
   3           2.55%           +0.04%
   4           1.09%           +0.00%

STATS_CHANGED

Change-Id: I0e344f8284a844be95d6953b431c839172019d09

av1/encoder/block.h

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 83c4960..f8d479a 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -413,6 +413,7 @@
   int must_find_valid_partition;
   int recalc_luma_mc_data;  // Flag to indicate recalculation of MC data during
                             // interpolation filter search
+  int use_transform_domain_distortion;
   // The likelihood of an edge existing in the block (using partial Canny edge
   // detection). For reference, 556 is the value returned for a solid
   // vertical black/white edge.

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 885d3a7..d005a16 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3146,7 +3146,7 @@
   // Tranform domain distortion is accurate for higher residuals.
   // TODO(any): Experiment with variance and mean based thresholds
   int use_transform_domain_distortion =
-      (cpi->sf.use_transform_domain_distortion > 0) &&
+      (x->use_transform_domain_distortion > 0) &&
       (block_mse_q8 >= cpi->tx_domain_dist_threshold) &&
       // Any 64-pt transforms only preserves half the coefficients.
       // Therefore transform domain distortion is not valid for these
@@ -3156,7 +3156,7 @@
   if (x->using_dist_8x8) use_transform_domain_distortion = 0;
 #endif
   int calc_pixel_domain_distortion_final =
-      cpi->sf.use_transform_domain_distortion == 1 &&
+      x->use_transform_domain_distortion == 1 &&
       use_transform_domain_distortion && x->rd_model != LOW_TXFM_RD;
   if (calc_pixel_domain_distortion_final &&
       (txk_allowed < TX_TYPES || allowed_tx_mask == 0x0001))

av1/encoder/rdopt.h

diff --git a/av1/encoder/rdopt.h b/av1/encoder/rdopt.h
index 099855c..5c34e69 100644
--- a/av1/encoder/rdopt.h
+++ b/av1/encoder/rdopt.h
@@ -250,6 +250,21 @@
   x->tx_mode = select_tx_mode(cpi, x->tx_size_search_method);
 }
 
+static INLINE void set_tx_domain_dist_type(
+    const struct AV1_COMP *cpi, MACROBLOCK *x,
+    int enable_winner_mode_for_tx_domain_dist, int is_winner_mode) {
+  if (!enable_winner_mode_for_tx_domain_dist) {
+    x->use_transform_domain_distortion =
+        cpi->sf.use_transform_domain_distortion;
+    return;
+  }
+
+  if (is_winner_mode)
+    x->use_transform_domain_distortion = 0;
+  else
+    x->use_transform_domain_distortion = 2;
+}
+
 // Checks the conditions to enable winner mode processing
 static INLINE int is_winner_mode_processing_enabled(
     const struct AV1_COMP *cpi, MB_MODE_INFO *const mbmi,
@@ -288,6 +303,8 @@
     case DEFAULT_EVAL:
       x->use_default_inter_tx_type = 0;
       x->use_default_intra_tx_type = 0;
+      // Set default transform domain distortion type
+      set_tx_domain_dist_type(cpi, x, 0, 0);
 
       // Get default threshold for R-D optimization of coefficients
       x->coeff_opt_dist_threshold =
@@ -302,6 +319,10 @@
       x->use_default_inter_tx_type =
           cpi->sf.tx_type_search.fast_inter_tx_type_search;
 
+      // Set transform domain distortion type for mode evaluation
+      set_tx_domain_dist_type(cpi, x,
+                              sf->enable_winner_mode_for_use_tx_domain_dist, 0);
+
       // Get threshold for R-D optimization of coefficients during mode
       // evaluation
       x->coeff_opt_dist_threshold =
@@ -315,6 +336,10 @@
       x->use_default_inter_tx_type = 0;
       x->use_default_intra_tx_type = 0;
 
+      // Set transform domain distortion type for winner mode evaluation
+      set_tx_domain_dist_type(cpi, x,
+                              sf->enable_winner_mode_for_use_tx_domain_dist, 1);
+
       // Get threshold for R-D optimization of coefficients for winner mode
       // evaluation
       x->coeff_opt_dist_threshold =

av1/encoder/speed_features.c

diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 7eddc50..29d87b7 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -377,6 +377,8 @@
     // TODO(any): Experiment with this speed feature by enabling for key frames
     sf->enable_winner_mode_for_tx_size_srch =
         frame_is_intra_only(&cpi->common) ? 0 : 1;
+    sf->enable_winner_mode_for_use_tx_domain_dist =
+        cm->allow_screen_content_tools ? 0 : 1;
     sf->reduce_wiener_window_size = is_boosted_arf2_bwd_type ? 0 : 1;
     sf->mv.subpel_search_method = SUBPEL_TREE_PRUNED;
     sf->simple_motion_search_prune_agg = 1;
@@ -858,6 +860,7 @@
   sf->perform_coeff_opt = 0;
   sf->enable_winner_mode_for_coeff_opt = 0;
   sf->enable_winner_mode_for_tx_size_srch = 0;
+  sf->enable_winner_mode_for_use_tx_domain_dist = 0;
   sf->prune_comp_type_by_model_rd = 0;
   sf->disable_smooth_intra = 0;
   sf->perform_best_rd_based_gating_for_chroma = 0;

av1/encoder/speed_features.h

diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 008bc61..1338cf9 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -681,6 +681,10 @@
   // search method
   int enable_winner_mode_for_tx_size_srch;
 
+  // Flag used to control the winner mode processing for use transform
+  // domain distortion
+  int enable_winner_mode_for_use_tx_domain_dist;
+
   // Flag used to control the speed of the eob selection in trellis.
   int trellis_eob_fast;