Combine sfs related to txfm_rd_gate_level

This CL combines speed features txfm_rd_gate_level and
motion_mode_txfm_rd_gating_offset.

Change-Id: Ie01ca81c85abc39485571c3c3dd8807118a358ee
diff --git a/av1/encoder/compound_type.c b/av1/encoder/compound_type.c
index 428eb76..b8bd164 100644
--- a/av1/encoder/compound_type.c
+++ b/av1/encoder/compound_type.c
@@ -1023,12 +1023,15 @@
                                         const BLOCK_SIZE bsize,
                                         int64_t ref_skip_rd, int mode_rate) {
   int eval_txfm = 1;
+  const int txfm_rd_gate_level =
+      get_txfm_rd_gate_level(cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
+                             TX_SEARCH_DEFAULT, /*eval_motion_mode=*/0);
   // Check if the mode is good enough based on skip rd
-  if (cpi->sf.inter_sf.txfm_rd_gate_level) {
+  if (txfm_rd_gate_level) {
     int64_t sse_y = compute_sse_plane(x, xd, PLANE_TYPE_Y, bsize);
     int64_t skip_rd = RDCOST(x->rdmult, mode_rate, (sse_y << 4));
-    eval_txfm = check_txfm_eval(x, bsize, ref_skip_rd, skip_rd,
-                                cpi->sf.inter_sf.txfm_rd_gate_level, 1);
+    eval_txfm =
+        check_txfm_eval(x, bsize, ref_skip_rd, skip_rd, txfm_rd_gate_level, 1);
   }
   return eval_txfm;
 }
@@ -1104,9 +1107,12 @@
   // Check if the mode is good enough based on skip rd
   // TODO(nithya): Handle wedge_newmv_search if extending for lower speed
   // setting
-  if (cpi->sf.inter_sf.txfm_rd_gate_level) {
+  const int txfm_rd_gate_level =
+      get_txfm_rd_gate_level(cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
+                             TX_SEARCH_DEFAULT, /*eval_motion_mode=*/0);
+  if (txfm_rd_gate_level) {
     int eval_txfm = check_txfm_eval(x, bsize, ref_skip_rd, skip_rd_cur,
-                                    cpi->sf.inter_sf.txfm_rd_gate_level, 1);
+                                    txfm_rd_gate_level, 1);
     if (!eval_txfm) {
       *comp_model_rd_cur = INT64_MAX;
       return INT64_MAX;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 9fc255d..ee15bd6 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1321,15 +1321,9 @@
   const int mi_row = xd->mi_row;
   const int mi_col = xd->mi_col;
   int mode_index_start, mode_index_end;
-  int txfm_rd_gate_level = cpi->sf.inter_sf.txfm_rd_gate_level;
-
-  // Set aggressive level of transform rd gating for mode evaluation stage.
-  if (cpi->sf.inter_sf.motion_mode_txfm_rd_gating_offset &&
-      txfm_rd_gate_level && !eval_motion_mode &&
-      num_pels_log2_lookup[bsize] > 8) {
-    txfm_rd_gate_level += cpi->sf.inter_sf.motion_mode_txfm_rd_gating_offset;
-    txfm_rd_gate_level = AOMMIN(txfm_rd_gate_level, MAX_TX_RD_GATE_LEVEL);
-  }
+  const int txfm_rd_gate_level =
+      get_txfm_rd_gate_level(cpi->sf.inter_sf.txfm_rd_gate_level, bsize,
+                             TX_SEARCH_MOTION_MODE, eval_motion_mode);
 
   // Modify the start and end index according to speed features. For example,
   // if SIMPLE_TRANSLATION has already been searched according to
@@ -5200,13 +5194,15 @@
     RD_STATS rd_stats_uv;
     const int mode_rate = inter_modes_info->mode_rate_arr[data_idx];
     int64_t skip_rd = INT64_MAX;
-    if (cpi->sf.inter_sf.txfm_rd_gate_level) {
+    const int txfm_rd_gate_level = get_txfm_rd_gate_level(
+        cpi->sf.inter_sf.txfm_rd_gate_level, bsize, TX_SEARCH_DEFAULT,
+        /*eval_motion_mode=*/0);
+    if (txfm_rd_gate_level) {
       // Check if the mode is good enough based on skip RD
       int64_t curr_sse = inter_modes_info->sse_arr[data_idx];
       skip_rd = RDCOST(x->rdmult, mode_rate, curr_sse);
-      int eval_txfm =
-          check_txfm_eval(x, bsize, search_state->best_skip_rd[0], skip_rd,
-                          cpi->sf.inter_sf.txfm_rd_gate_level, 0);
+      int eval_txfm = check_txfm_eval(x, bsize, search_state->best_skip_rd[0],
+                                      skip_rd, txfm_rd_gate_level, 0);
       if (!eval_txfm) continue;
     }
 
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h
index 17e76fb..1c5b3db 100644
--- a/av1/encoder/rdopt_utils.h
+++ b/av1/encoder/rdopt_utils.h
@@ -772,6 +772,18 @@
          USABLE_REF_MV_STACK_SIZE * sizeof(xd->ref_mv_stack[0][0]));
 }
 
+// Get transform rd gate level for the given transform search case.
+static INLINE int get_txfm_rd_gate_level(
+    const int txfm_rd_gate_level[TX_SEARCH_CASES], BLOCK_SIZE bsize,
+    TX_SEARCH_CASE tx_search_case, int eval_motion_mode) {
+  assert(tx_search_case < TX_SEARCH_CASES);
+  if (tx_search_case == TX_SEARCH_MOTION_MODE && !eval_motion_mode &&
+      num_pels_log2_lookup[bsize] > 8)
+    return txfm_rd_gate_level[TX_SEARCH_MOTION_MODE];
+
+  return txfm_rd_gate_level[TX_SEARCH_DEFAULT];
+}
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index ce5305e..3765d77 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -168,6 +168,14 @@
   return frame_is_kf_gf_arf(cpi);
 }
 
+// Set transform rd gate level for all transform search cases.
+static AOM_INLINE void set_txfm_rd_gate_level(
+    int txfm_rd_gate_level[TX_SEARCH_CASES], int level) {
+  assert(level <= MAX_TX_RD_GATE_LEVEL);
+  for (int idx = 0; idx < TX_SEARCH_CASES; idx++)
+    txfm_rd_gate_level[idx] = level;
+}
+
 static void set_allintra_speed_feature_framesize_dependent(
     const AV1_COMP *const cpi, SPEED_FEATURES *const sf, int speed) {
   const AV1_COMMON *const cm = &cpi->common;
@@ -1045,7 +1053,7 @@
     sf->inter_sf.use_dist_wtd_comp_flag = DIST_WTD_COMP_DISABLED;
     sf->inter_sf.enable_fast_compound_mode_search = 1;
     sf->inter_sf.reuse_mask_search_results = 1;
-    sf->inter_sf.txfm_rd_gate_level = boosted ? 0 : 1;
+    set_txfm_rd_gate_level(sf->inter_sf.txfm_rd_gate_level, boosted ? 0 : 1);
     sf->inter_sf.inter_mode_txfm_breakout = boosted ? 0 : 1;
     sf->inter_sf.alt_ref_search_fp = 1;
 
@@ -1105,8 +1113,8 @@
     sf->inter_sf.prune_comp_search_by_single_result = boosted ? 4 : 2;
     sf->inter_sf.selective_ref_frame = 5;
     sf->inter_sf.reuse_compound_type_decision = 1;
-    sf->inter_sf.txfm_rd_gate_level =
-        boosted ? 0 : (is_boosted_arf2_bwd_type ? 1 : 2);
+    set_txfm_rd_gate_level(sf->inter_sf.txfm_rd_gate_level,
+                           boosted ? 0 : (is_boosted_arf2_bwd_type ? 1 : 2));
     sf->inter_sf.inter_mode_txfm_breakout = boosted ? 0 : 2;
 
     sf->interp_sf.adaptive_interp_filter_search = 2;
@@ -1164,8 +1172,8 @@
                                                                           : 1;
 
     sf->inter_sf.alt_ref_search_fp = 2;
-    sf->inter_sf.txfm_rd_gate_level = boosted ? 0 : 3;
-    sf->inter_sf.motion_mode_txfm_rd_gating_offset = boosted ? 0 : 2;
+    sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_DEFAULT] = boosted ? 0 : 3;
+    sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_MOTION_MODE] = boosted ? 0 : 5;
 
     sf->inter_sf.prune_inter_modes_based_on_tpl = boosted ? 0 : 2;
     sf->inter_sf.prune_ext_comp_using_neighbors = 2;
@@ -1222,7 +1230,7 @@
     sf->mv_sf.warp_search_method = WARP_SEARCH_DIAMOND;
 
     sf->inter_sf.prune_inter_modes_if_skippable = 1;
-    sf->inter_sf.txfm_rd_gate_level = boosted ? 0 : 4;
+    sf->inter_sf.txfm_rd_gate_level[TX_SEARCH_DEFAULT] = boosted ? 0 : 4;
     sf->inter_sf.enable_fast_compound_mode_search = 2;
 
     sf->intra_sf.chroma_intra_pruning_with_hog = 3;
@@ -1631,7 +1639,7 @@
   sf->inter_sf.disable_interintra_wedge_var_thresh = UINT_MAX;
   sf->inter_sf.selective_ref_frame = 4;
   sf->inter_sf.alt_ref_search_fp = 2;
-  sf->inter_sf.txfm_rd_gate_level = boosted ? 0 : 4;
+  set_txfm_rd_gate_level(sf->inter_sf.txfm_rd_gate_level, boosted ? 0 : 4);
   sf->inter_sf.limit_txfm_eval_per_mode = 3;
 
   sf->inter_sf.adaptive_rd_thresh = 4;
@@ -2011,8 +2019,6 @@
   inter_sf->prune_ref_mv_idx_search = 0;
   inter_sf->prune_warped_prob_thresh = 0;
   inter_sf->reuse_compound_type_decision = 0;
-  inter_sf->txfm_rd_gate_level = 0;
-  inter_sf->motion_mode_txfm_rd_gating_offset = 0;
   inter_sf->prune_inter_modes_if_skippable = 0;
   inter_sf->disable_masked_comp = 0;
   inter_sf->enable_fast_compound_mode_search = 0;
@@ -2022,6 +2028,7 @@
   inter_sf->limit_inter_mode_cands = 0;
   inter_sf->limit_txfm_eval_per_mode = 0;
   inter_sf->skip_arf_compound = 0;
+  set_txfm_rd_gate_level(inter_sf->txfm_rd_gate_level, 0);
 }
 
 static AOM_INLINE void init_interp_sf(INTERP_FILTER_SPEED_FEATURES *interp_sf) {
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 2392a35..0bdfaa3 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -234,6 +234,16 @@
   PRUNE_NEARMV_MAX = PRUNE_NEARMV_LEVEL3,
 } UENUM1BYTE(PRUNE_NEARMV_LEVEL);
 
+enum {
+  // Default Transform search case - used in evaluation of compound type mode
+  // and best inter candidates
+  TX_SEARCH_DEFAULT = 0,
+  // Transform search in motion mode rd
+  TX_SEARCH_MOTION_MODE,
+  // All transform search cases
+  TX_SEARCH_CASES
+} UENUM1BYTE(TX_SEARCH_CASE);
+
 typedef struct {
   TX_TYPE_PRUNE_MODE prune_2d_txfm_mode;
   int fast_intra_tx_type_search;
@@ -855,13 +865,8 @@
   // Bypass transform search based on skip rd at following stages
   //   i. Compound type mode search
   //  ii. Motion mode search (mode evaluation and winner motion mode stage)
-  // iii. Trasform search for best inter candidates
-  int txfm_rd_gate_level;
-
-  // Set transform rd gating offset used in mode evaluation stage of motion
-  // mode. This sf is applicable only if txfm_rd_gate_level is enabled.
-  // TODO(Cherma): Combine this sf with txfm_rd_gate_level.
-  int motion_mode_txfm_rd_gating_offset;
+  // iii. Transform search for best inter candidates
+  int txfm_rd_gate_level[TX_SEARCH_CASES];
 
   // Limit the inter mode tested in the RD loop
   int reduce_inter_modes;