More strict rule to decide if intrabc should be on

In addition to the number of colors in a block, consider also the
variance of the block to decide if intrabc should be enabled. This
can prevent mistakenly predicting some smooth areas as screen content,
where intrabc should actually be disabled to allow loop filters.

Tested keyframe encoding(20 frames) on speed 1:
lowres            -0.06%(-1.64% on flower; -0.75% on mobisode2)
midres            -0.22%(-4.43 on modisode2; -2.28% on snow_mnt)
screen_content    -0.15%(-2.19% on liquify)

BUG=aomedia:2319

STATS_CHANGED

Change-Id: If1d0417785a32a6e8ad85757d690645285c802b6

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 28b72d9..487b3ac 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6347,8 +6347,6 @@
   av1_zero(*td->counts);
   av1_zero(rdc->comp_pred_diff);
 
-  // Allow intrabc when screen content tools are enabled.
-  cm->allow_intrabc = cm->allow_screen_content_tools;
   // Reset the flag.
   cpi->intrabc_used = 0;
   // Need to disable intrabc when superres is selected

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 130aa9f..9f61585 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -99,30 +99,6 @@
 #define FILE_NAME_LEN 100
 #endif
 
-// Estimate if the source frame is screen content, based on the portion of
-// blocks that have no more than 4 (experimentally selected) luma colors.
-static int is_screen_content(const uint8_t *src, int use_hbd, int bd,
-                             int stride, int width, int height) {
-  assert(src != NULL);
-  int counts = 0;
-  const int blk_w = 16;
-  const int blk_h = 16;
-  const int limit = 4;
-  for (int r = 0; r + blk_h <= height; r += blk_h) {
-    for (int c = 0; c + blk_w <= width; c += blk_w) {
-      int count_buf[1 << 12];  // Maximum (1 << 12) color levels.
-      const int n_colors =
-          use_hbd ? av1_count_colors_highbd(src + r * stride + c, stride, blk_w,
-                                            blk_h, bd, count_buf)
-                  : av1_count_colors(src + r * stride + c, stride, blk_w, blk_h,
-                                     count_buf);
-      if (n_colors > 1 && n_colors <= limit) counts++;
-    }
-  }
-  // The threshold is 10%.
-  return counts * blk_h * blk_w * 10 > width * height;
-}
-
 static INLINE void Scale2Ratio(AOM_SCALING mode, int *hr, int *hs) {
   switch (mode) {
     case NORMAL:
@@ -3493,6 +3469,71 @@
   }
 }
 
+static void set_screen_content_options(AV1_COMP *cpi) {
+  AV1_COMMON *cm = &cpi->common;
+
+  if (cm->seq_params.force_screen_content_tools != 2) {
+    cm->allow_screen_content_tools = cm->allow_intrabc =
+        cm->seq_params.force_screen_content_tools;
+    return;
+  }
+
+  if (cpi->oxcf.content == AOM_CONTENT_SCREEN) {
+    cm->allow_screen_content_tools = cm->allow_intrabc = 1;
+    return;
+  }
+
+  // Estimate if the source frame is screen content, based on the portion of
+  // blocks that have few luma colors.
+  const uint8_t *src = cpi->source->y_buffer;
+  assert(src != NULL);
+  const int use_hbd = cpi->source->flags & YV12_FLAG_HIGHBITDEPTH;
+  const int stride = cpi->source->y_stride;
+  const int width = cpi->source->y_width;
+  const int height = cpi->source->y_height;
+  const int bd = cm->seq_params.bit_depth;
+  const int blk_w = 16;
+  const int blk_h = 16;
+  // These threshold values are selected experimentally.
+  const int color_thresh = 4;
+  const unsigned int var_thresh = 0;
+  // Counts of blocks with no more than color_thresh colors.
+  int counts_1 = 0;
+  // Counts of blocks with no more than color_thresh colors and variance larger
+  // than var_thresh.
+  int counts_2 = 0;
+
+  for (int r = 0; r + blk_h <= height; r += blk_h) {
+    for (int c = 0; c + blk_w <= width; c += blk_w) {
+      int count_buf[1 << 12];  // Maximum (1 << 12) color levels.
+      const uint8_t *const this_src = src + r * stride + c;
+      const int n_colors =
+          use_hbd ? av1_count_colors_highbd(this_src, stride, blk_w, blk_h, bd,
+                                            count_buf)
+                  : av1_count_colors(this_src, stride, blk_w, blk_h, count_buf);
+      if (n_colors > 1 && n_colors <= color_thresh) {
+        ++counts_1;
+        struct buf_2d buf;
+        buf.stride = stride;
+        buf.buf = (uint8_t *)this_src;
+        const unsigned int var =
+            use_hbd
+                ? av1_high_get_sby_perpixel_variance(cpi, &buf, BLOCK_16X16, bd)
+                : av1_get_sby_perpixel_variance(cpi, &buf, BLOCK_16X16);
+        if (var > var_thresh) ++counts_2;
+      }
+    }
+  }
+
+  // The threshold values are selected experimentally.
+  cm->allow_screen_content_tools =
+      counts_1 * blk_h * blk_w * 10 > width * height;
+  // IntraBC would force loop filters off, so we use more strict rules that also
+  // requires that the block has high variance.
+  cm->allow_intrabc = cm->allow_screen_content_tools &&
+                      counts_2 * blk_h * blk_w * 15 > width * height;
+}
+
 static void set_size_independent_vars(AV1_COMP *cpi) {
   int i;
   AV1_COMMON *cm = &cpi->common;
@@ -3505,19 +3546,7 @@
   cm->interp_filter = SWITCHABLE;
   cm->switchable_motion_mode = 1;
 
-  if (frame_is_intra_only(cm)) {
-    if (cm->seq_params.force_screen_content_tools == 2) {
-      cm->allow_screen_content_tools =
-          cpi->oxcf.content == AOM_CONTENT_SCREEN ||
-          is_screen_content(cpi->source->y_buffer,
-                            cpi->source->flags & YV12_FLAG_HIGHBITDEPTH,
-                            cm->seq_params.bit_depth, cpi->source->y_stride,
-                            cpi->source->y_width, cpi->source->y_height);
-    } else {
-      cm->allow_screen_content_tools =
-          cm->seq_params.force_screen_content_tools;
-    }
-  }
+  if (frame_is_intra_only(cm)) set_screen_content_options(cpi);
   cpi->is_screen_content_type = (cm->allow_screen_content_tools != 0);
 }