Adjust altref filter strength based on noise level
Uses a simple noise estimator for altref filtering to
modulate the filtering strength.
lowres 30 frames, q mode: -0.05%, not much change in speed
Experiments to be continued with boosts and other parameters.
STATS_CHANGED
Change-Id: Iad5ab781470149ef429dde20e38f46327e934000
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index 374ea23..cab4b73 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -34,6 +34,9 @@
#include "aom_ports/aom_timer.h"
#include "aom_scale/aom_scale.h"
+#define EDGE_THRESHOLD 50
+#define SQRT_PI_BY_2 1.25331413732
+
static void temporal_filter_predictors_mb_c(
MACROBLOCKD *xd, uint8_t *y_mb_ptr, uint8_t *u_mb_ptr, uint8_t *v_mb_ptr,
int stride, int uv_block_width, int uv_block_height, int mv_row, int mv_col,
@@ -493,6 +496,83 @@
for (i = 0; i < num_planes; i++) mbd->plane[i].pre[0].buf = input_buffer[i];
}
+// This is an adaptation of the mehtod in the following paper:
+// Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise
+// estimation using Laplacian operator and adaptive edge detection,"
+// Proc. 3rd International Symposium on Communications, Control and
+// Signal Processing, 2008, St Julians, Malta.
+//
+// Return noise estimate, or -1.0 if there was a failure
+static double estimate_noise(const uint8_t *src, int width, int height,
+ int stride, int edge_thresh) {
+ int64_t sum = 0;
+ int64_t num = 0;
+ for (int i = 1; i < height - 1; ++i) {
+ for (int j = 1; j < width - 1; ++j) {
+ const int k = i * stride + j;
+ // Sobel gradients
+ const int Gx = (src[k - stride - 1] - src[k - stride + 1]) +
+ (src[k + stride - 1] - src[k + stride + 1]) +
+ 2 * (src[k - 1] - src[k + 1]);
+ const int Gy = (src[k - stride - 1] - src[k + stride - 1]) +
+ (src[k - stride + 1] - src[k + stride + 1]) +
+ 2 * (src[k - stride] - src[k + stride]);
+ const int Ga = abs(Gx) + abs(Gy);
+ if (Ga < edge_thresh) { // Smooth pixels
+ // Find Laplacian
+ const int v =
+ 4 * src[k] -
+ 2 * (src[k - 1] + src[k + 1] + src[k - stride] + src[k + stride]) +
+ (src[k - stride - 1] + src[k - stride + 1] + src[k + stride - 1] +
+ src[k + stride + 1]);
+ sum += abs(v);
+ ++num;
+ }
+ }
+ }
+ // If very few smooth pels, return -1 since the estimate is unreliable
+ if (num < 16) return -1.0;
+
+ const double sigma = (double)sum / (6 * num) * SQRT_PI_BY_2;
+ return sigma;
+}
+
+// Return noise estimate, or -1.0 if there was a failure
+static double highbd_estimate_noise(const uint8_t *src8, int width, int height,
+ int stride, int bd, int edge_thresh) {
+ uint16_t *src = CONVERT_TO_SHORTPTR(src8);
+ int64_t sum = 0;
+ int64_t num = 0;
+ for (int i = 1; i < height - 1; ++i) {
+ for (int j = 1; j < width - 1; ++j) {
+ const int k = i * stride + j;
+ // Sobel gradients
+ const int Gx = (src[k - stride - 1] - src[k - stride + 1]) +
+ (src[k + stride - 1] - src[k + stride + 1]) +
+ 2 * (src[k - 1] - src[k + 1]);
+ const int Gy = (src[k - stride - 1] - src[k + stride - 1]) +
+ (src[k - stride + 1] - src[k + stride + 1]) +
+ 2 * (src[k - stride] - src[k + stride]);
+ const int Ga = ROUND_POWER_OF_TWO(abs(Gx) + abs(Gy), bd - 8);
+ if (Ga < edge_thresh) { // Smooth pixels
+ // Find Laplacian
+ const int v =
+ 4 * src[k] -
+ 2 * (src[k - 1] + src[k + 1] + src[k - stride] + src[k + stride]) +
+ (src[k - stride - 1] + src[k - stride + 1] + src[k + stride - 1] +
+ src[k + stride + 1]);
+ sum += ROUND_POWER_OF_TWO(abs(v), bd - 8);
+ ++num;
+ }
+ }
+ }
+ // If very few smooth pels, return -1 since the estimate is unreliable
+ if (num < 16) return -1.0;
+
+ const double sigma = (double)sum / (6 * num) * SQRT_PI_BY_2;
+ return sigma;
+}
+
// Apply buffer limits and context specific adjustments to arnr filter.
static void adjust_arnr_filter(AV1_COMP *cpi, int distance, int group_boost,
int *arnr_frames, int *arnr_strength) {
@@ -523,10 +603,38 @@
else
q = ((int)av1_convert_qindex_to_q(cpi->rc.avg_frame_qindex[KEY_FRAME],
cpi->common.seq_params.bit_depth));
- if (q > 16) {
- strength = oxcf->arnr_strength;
+ MACROBLOCKD *mbd = &cpi->td.mb.e_mbd;
+ struct lookahead_entry *buf = av1_lookahead_peek(cpi->lookahead, distance);
+ double noiselevel;
+ if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
+ noiselevel = highbd_estimate_noise(
+ buf->img.y_buffer, buf->img.y_crop_width, buf->img.y_crop_height,
+ buf->img.y_stride, mbd->bd, EDGE_THRESHOLD);
} else {
- strength = oxcf->arnr_strength - ((16 - q) / 2);
+ noiselevel = estimate_noise(buf->img.y_buffer, buf->img.y_crop_width,
+ buf->img.y_crop_height, buf->img.y_stride,
+ EDGE_THRESHOLD);
+ }
+ int adj_strength = oxcf->arnr_strength;
+ if (noiselevel > 0) {
+ // Get 4 integer adjustment levels in [-2, 1]
+ int noiselevel_adj;
+ if (noiselevel < 0.75)
+ noiselevel_adj = -2;
+ else if (noiselevel < 1.75)
+ noiselevel_adj = -1;
+ else if (noiselevel < 4.0)
+ noiselevel_adj = 0;
+ else
+ noiselevel_adj = 1;
+ adj_strength += noiselevel_adj;
+ }
+ // printf("[noise level: %g, strength = %d]\n", noiselevel, adj_strength);
+
+ if (q > 16) {
+ strength = adj_strength;
+ } else {
+ strength = adj_strength - ((16 - q) / 2);
if (strength < 0) strength = 0;
}
@@ -557,14 +665,16 @@
const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
// Apply context specific adjustments to the arnr filter parameters.
- adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, &strength);
- // TODO(weitinglin): Currently, we enforce the filtering strength on
- // extra ARFs' to be zeros. We should investigate in which
- // case it is more beneficial to use non-zero strength
- // filtering.
if (gf_group->update_type[gf_group->index] == INTNL_ARF_UPDATE) {
+ // TODO(weitinglin): Currently, we enforce the filtering strength on
+ // extra ARFs' to be zeros. We should investigate in which
+ // case it is more beneficial to use non-zero strength
+ // filtering.
strength = 0;
frames_to_blur = 1;
+ } else {
+ adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur,
+ &strength);
}
int which_arf = gf_group->arf_update_idx[gf_group->index];
