tx skip prediction: add pre-screening on prediction error

Exit early on large prediction error. This reduces the overhead of the
tx skip prediction, especially for low quantizers.

BUG=aomedia:1106

Change-Id: Icad8e01cdeb2e8f4cf0befa7f5a89e088f3c17e5

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 8e6835f..8b14613 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4727,7 +4727,8 @@
 
 // Uses simple features on top of DCT coefficients to quickly predict
 // whether optimal RD decision is to skip encoding the residual.
-static int predict_skip_flag(const MACROBLOCK *x, BLOCK_SIZE bsize) {
+// The sse value is stored in dist.
+static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist) {
   const int max_tx_size =
       get_max_rect_tx_size(bsize, is_inter_block(&x->e_mbd.mi[0]->mbmi));
   const int tx_h = tx_size_high[max_tx_size];
@@ -4737,6 +4738,17 @@
   const int bw = block_size_wide[bsize];
   const int bh = block_size_high[bsize];
   const MACROBLOCKD *xd = &x->e_mbd;
+  const uint32_t dc_q = (uint32_t)av1_dc_quant_QTX(x->qindex, 0, xd->bd);
+
+  *dist = pixel_diff_dist(x, 0, x->plane[0].src_diff, bw, 0, 0, bsize, bsize);
+  const int64_t mse = *dist / bw / bh;
+  // Normalized quantizer takes the transform upscaling factor (8 for tx size
+  // smaller than 32) into account.
+  const uint32_t normalized_dc_q = dc_q >> 3;
+  const int64_t mse_thresh = (int64_t)normalized_dc_q * normalized_dc_q / 8;
+  // Predict not to skip when mse is larger than threshold.
+  if (mse > mse_thresh) return 0;
+
   DECLARE_ALIGNED(32, tran_low_t, DCT_coefs[32 * 32]);
   TxfmParam param;
   param.tx_type = DCT_DCT;
@@ -4752,8 +4764,7 @@
   // within this function.
   param.tx_set_type = get_ext_tx_set_type(param.tx_size, plane_bsize,
                                           is_inter_block(&xd->mi[0]->mbmi), 0);
-  const uint32_t dc = (uint32_t)av1_dc_quant_QTX(x->qindex, 0, xd->bd);
-  const uint32_t ac = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd);
+  const uint32_t ac_q = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd);
   uint32_t max_quantized_coef = 0;
   const int16_t *src_diff = x->plane[0].src_diff;
   for (int row = 0; row < bh; row += tx_h) {
@@ -4770,7 +4781,7 @@
       // Operating on TX domain, not pixels; we want the QTX quantizers
       for (int i = 0; i < tx_w * tx_h; ++i) {
         uint32_t cur_quantized_coef =
-            (100 * (uint32_t)abs(DCT_coefs[i])) / (i ? ac : dc);
+            (100 * (uint32_t)abs(DCT_coefs[i])) / (i ? ac_q : dc_q);
         if (cur_quantized_coef > max_quantized_coef)
           max_quantized_coef = cur_quantized_coef;
       }
@@ -4783,7 +4794,7 @@
 
 // Used to set proper context for early termination with skip = 1.
 static void set_skip_flag(const AV1_COMP *cpi, MACROBLOCK *x,
-                          RD_STATS *rd_stats, int bsize) {
+                          RD_STATS *rd_stats, int bsize, int64_t dist) {
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
   const int n4 = bsize_to_num_blk(bsize);
@@ -4838,15 +4849,11 @@
   }
 #endif  // CONFIG_TXK_SEL
   rd_stats->rate = rate;
-
-  // Distortion.
-  int64_t tmp = pixel_diff_dist(x, 0, x->plane[0].src_diff,
-                                block_size_wide[bsize], 0, 0, bsize, bsize);
 #if CONFIG_HIGHBITDEPTH
   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
-    tmp = ROUND_POWER_OF_TWO(tmp, (xd->bd - 8) * 2);
+    dist = ROUND_POWER_OF_TWO(dist, (xd->bd - 8) * 2);
 #endif  // CONFIG_HIGHBITDEPTH
-  rd_stats->dist = rd_stats->sse = (tmp << 4);
+  rd_stats->dist = rd_stats->sse = (dist << 4);
 }
 
 static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
@@ -4900,9 +4907,10 @@
 
   // If we predict that skip is the optimal RD decision - set the respective
   // context and terminate early.
+  int64_t dist;
   if (is_inter && cpi->sf.tx_type_search.use_skip_flag_prediction &&
-      predict_skip_flag(x, bsize)) {
-    set_skip_flag(cpi, x, rd_stats, bsize);
+      predict_skip_flag(x, bsize, &dist)) {
+    set_skip_flag(cpi, x, rd_stats, bsize, dist);
     // Save the RD search results into tx_rd_record.
     if (within_border) save_tx_rd_info(n4, hash, x, rd_stats, tx_rd_record);
     return;