Refactor pred_dual_interp_filter_rd function

Introduced a generic function find_best_interp_rd_facade()
for interp filters evaluation.

Introduced allowed_interp_mask set based on lookup table
for dual interpolation filter.

Change-Id: Iadb66942ed1d9cf68b8470740ccaad0226c969ac
diff --git a/av1/common/filter.h b/av1/common/filter.h
index d6a86a7..7ae4672 100644
--- a/av1/common/filter.h
+++ b/av1/common/filter.h
@@ -19,6 +19,7 @@
 #include "aom/aom_integer.h"
 #include "aom_dsp/aom_filter.h"
 #include "aom_ports/mem.h"
+#include "av1/common/enums.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -57,6 +58,7 @@
   INTERP_HORZ_EQ_VERT_NEQ,
   INTERP_HORZ_NEQ_VERT_EQ,
   INTERP_HORZ_EQ_VERT_EQ,
+  INTERP_PRED_TYPE_ALL,
 } UENUM1BYTE(INTERP_PRED_TYPE);
 // Pack two InterpFilter's into a uint32_t: since there are at most 10 filters,
 // we can use 16 bits for each and have more than enough space. This reduces
@@ -96,6 +98,7 @@
 #define SWITCHABLE_FILTER_CONTEXTS ((SWITCHABLE_FILTERS + 1) * 4)
 #define INTER_FILTER_COMP_OFFSET (SWITCHABLE_FILTERS + 1)
 #define INTER_FILTER_DIR_OFFSET ((SWITCHABLE_FILTERS + 1) * 2)
+#define ALLOW_ALL_INTERP_FILT_MASK (0x01ff)
 
 typedef struct InterpFilterParams {
   const int16_t *filter_ptr;
@@ -198,6 +201,16 @@
   { 0, 0, 4, 36, 62, 26, 0, 0 },  { 0, 0, 2, 34, 62, 30, 0, 0 }
 };
 
+static const uint16_t
+    av1_interp_dual_filt_mask[INTERP_PRED_TYPE_ALL - 2][SWITCHABLE_FILTERS] = {
+      { (1 << REG_REG) | (1 << SMOOTH_REG) | (1 << SHARP_REG),
+        (1 << REG_SMOOTH) | (1 << SMOOTH_SMOOTH) | (1 << SHARP_SMOOTH),
+        (1 << REG_SHARP) | (1 << SMOOTH_SHARP) | (1 << SHARP_SHARP) },
+      { (1 << REG_REG) | (1 << REG_SMOOTH) | (1 << REG_SHARP),
+        (1 << SMOOTH_REG) | (1 << SMOOTH_SMOOTH) | (1 << SMOOTH_SHARP),
+        (1 << SHARP_REG) | (1 << SHARP_SMOOTH) | (1 << SHARP_SHARP) }
+    };
+
 // For w<=4, MULTITAP_SHARP is the same as EIGHTTAP_REGULAR
 static const InterpFilterParams av1_interp_4tap[SWITCHABLE_FILTERS + 1] = {
   { (const int16_t *)av1_sub_pel_filters_4, SUBPEL_TAPS, SUBPEL_SHIFTS,
@@ -248,6 +261,22 @@
   }
 }
 
+static INLINE void reset_interp_filter_allowed_mask(
+    uint16_t *allow_interp_mask, DUAL_FILTER_TYPE filt_type) {
+  uint16_t tmp = ~(1 << filt_type);
+  *allow_interp_mask &= (tmp & ALLOW_ALL_INTERP_FILT_MASK);
+}
+
+static INLINE void set_interp_filter_allowed_mask(uint16_t *allow_interp_mask,
+                                                  DUAL_FILTER_TYPE filt_type) {
+  *allow_interp_mask |= (1 << filt_type);
+}
+
+static INLINE uint8_t get_interp_filter_allowed_mask(
+    uint16_t allow_interp_mask, DUAL_FILTER_TYPE filt_type) {
+  return (allow_interp_mask >> filt_type) & 1;
+}
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index d9b9526f..b3804b5 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -8310,6 +8310,36 @@
   return pred_filter_enable * pred_filter_type;
 }
 
+static DUAL_FILTER_TYPE find_best_interp_rd_facade(
+    MACROBLOCK *const x, const AV1_COMP *const cpi,
+    const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col,
+    const BUFFER_SET *const orig_dst, int64_t *const rd, RD_STATS *rd_stats_y,
+    RD_STATS *rd_stats, int *const switchable_rate,
+    const BUFFER_SET *dst_bufs[2], const int switchable_ctx[2],
+    const int skip_pred, uint16_t allow_interp_mask, int is_w4_or_h4) {
+  int tmp_skip_pred = skip_pred;
+  DUAL_FILTER_TYPE best_filt_type = REG_REG;
+
+  // If no filter are set to be evaluated, return from function
+  if (allow_interp_mask == 0x0) return best_filt_type;
+  // For block width or height is 4, skip the pred evaluation of SHARP_SHARP
+  tmp_skip_pred = is_w4_or_h4 ? cpi->default_interp_skip_flags : skip_pred;
+
+  // Loop over the all filter types and evaluate for only allowed filter types
+  for (int filt_type = SHARP_SHARP; filt_type >= REG_REG; --filt_type) {
+    const int is_filter_allowed =
+        get_interp_filter_allowed_mask(allow_interp_mask, filt_type);
+    if (is_filter_allowed)
+      if (interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col,
+                                  orig_dst, rd, rd_stats_y, rd_stats,
+                                  switchable_rate, dst_bufs, filt_type,
+                                  switchable_ctx, tmp_skip_pred))
+        best_filt_type = filt_type;
+    tmp_skip_pred = skip_pred;
+  }
+  return best_filt_type;
+}
+
 static INLINE void pred_dual_interp_filter_rd(
     MACROBLOCK *const x, const AV1_COMP *const cpi,
     const TileDataEnc *tile_data, BLOCK_SIZE bsize, int mi_row, int mi_col,
@@ -8319,48 +8349,30 @@
     const int skip_pred, INTERP_PRED_TYPE pred_filt_type, int_interpfilters *af,
     int_interpfilters *lf) {
   (void)lf;
-  int filter_idx = 0;
-  InterpFilter af_horiz = INTERP_INVALID, af_vert = INTERP_INVALID;
-  af_horiz = af->as_filters.x_filter;
-  af_vert = af->as_filters.y_filter;
-  assert(pred_filt_type != INTERP_HORZ_NEQ_VERT_NEQ);
+  assert(pred_filt_type > INTERP_HORZ_NEQ_VERT_NEQ);
+  assert(pred_filt_type < INTERP_PRED_TYPE_ALL);
+  uint16_t allowed_interp_mask = 0;
 
-  // pred_filter_search = 1: pred_filter is enabled and only horz pred matching
   if (pred_filt_type == INTERP_HORZ_EQ_VERT_NEQ) {
-    for (filter_idx = (DUAL_FILTER_SET_SIZE - SWITCHABLE_FILTERS + af_horiz);
-         filter_idx >= 0; filter_idx -= SWITCHABLE_FILTERS) {
-      if (filter_idx) {
-        interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col,
-                                orig_dst, rd, rd_stats_y, rd_stats,
-                                switchable_rate, dst_bufs, filter_idx,
-                                switchable_ctx, skip_pred);
-      }
-    }
+    // pred_filter_search = 1: Only horizontal filter is matching
+    allowed_interp_mask =
+        av1_interp_dual_filt_mask[pred_filt_type - 1][af->as_filters.x_filter];
   } else if (pred_filt_type == INTERP_HORZ_NEQ_VERT_EQ) {
-    // pred_filter_search = 2: pred_filter is enabled and
-    //                         only vert pred matching
-    for (filter_idx = (af_vert * SWITCHABLE_FILTERS + 2);
-         filter_idx >= ((af_vert * SWITCHABLE_FILTERS)); filter_idx -= 1) {
-      if (filter_idx) {
-        interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col,
-                                orig_dst, rd, rd_stats_y, rd_stats,
-                                switchable_rate, dst_bufs, filter_idx,
-                                switchable_ctx, skip_pred);
-      }
-    }
-  } else if (pred_filt_type == INTERP_HORZ_EQ_VERT_EQ) {
-    // pred_filter_search = 3: pred_filter is enabled and
-    //                         both vert, horz pred matching
-    filter_idx = af_horiz + (af_vert * SWITCHABLE_FILTERS);
-    if (filter_idx) {
-      interpolation_filter_rd(x, cpi, tile_data, bsize, mi_row, mi_col,
-                              orig_dst, rd, rd_stats_y, rd_stats,
-                              switchable_rate, dst_bufs, filter_idx,
-                              switchable_ctx, skip_pred);
-    }
+    // pred_filter_search = 2: Only vertical filter is matching
+    allowed_interp_mask =
+        av1_interp_dual_filt_mask[pred_filt_type - 1][af->as_filters.y_filter];
   } else {
-    assert(0);
+    // pred_filter_search = 3: Both horizontal and vertical filter are matching
+    int filt_type =
+        af->as_filters.x_filter + af->as_filters.y_filter * SWITCHABLE_FILTERS;
+    set_interp_filter_allowed_mask(&allowed_interp_mask, filt_type);
   }
+  // REG_REG is already been evaluated in the beginning
+  reset_interp_filter_allowed_mask(&allowed_interp_mask, REG_REG);
+  find_best_interp_rd_facade(x, cpi, tile_data, bsize, mi_row, mi_col, orig_dst,
+                             rd, rd_stats_y, rd_stats, switchable_rate,
+                             dst_bufs, switchable_ctx, skip_pred,
+                             allowed_interp_mask, 0);
 }
 // Evaluate dual filter type
 // a) Using above, left block interp filter