AV1 RT: Implement Noise Estimator
Turning this off for now. Overall the impact is small, but there are 5
outliers clips that show BDRate degradation of ~6%, with speed up on
these clips of ~15-20%. Need to investigate and tune furhter with
Denoiser
Change-Id: I9383c9f156cb514d290270e629c9ce0a259d4c9b
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 4886fe5..16cb440 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -234,6 +234,8 @@
"${AOM_ROOT}/av1/encoder/wedge_utils.c"
"${AOM_ROOT}/av1/encoder/var_based_part.c"
"${AOM_ROOT}/av1/encoder/var_based_part.h"
+ "${AOM_ROOT}/av1/encoder/av1_noise_estimate.c"
+ "${AOM_ROOT}/av1/encoder/av1_noise_estimate.h"
"${AOM_ROOT}/third_party/fastfeat/fast.c"
"${AOM_ROOT}/third_party/fastfeat/fast.h"
"${AOM_ROOT}/third_party/fastfeat/fast_9.c"
diff --git a/av1/encoder/av1_noise_estimate.c b/av1/encoder/av1_noise_estimate.c
new file mode 100644
index 0000000..edf1358
--- /dev/null
+++ b/av1/encoder/av1_noise_estimate.c
@@ -0,0 +1,305 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <assert.h>
+#include <limits.h>
+#include <math.h>
+
+#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_scale/yv12config.h"
+#include "aom/aom_integer.h"
+#include "av1/encoder/context_tree.h"
+#include "av1/encoder/av1_noise_estimate.h"
+#include "av1/encoder/encoder.h"
+
+#if CONFIG_AV1_TEMPORAL_DENOISING
+// For SVC: only do noise estimation on top spatial layer.
+static INLINE int noise_est_svc(const struct AV1_COMP *const cpi) {
+ return (!cpi->use_svc ||
+ (cpi->use_svc &&
+ cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1));
+}
+#endif
+
+void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) {
+ ne->enabled = 0;
+ ne->level = (width * height < 1280 * 720) ? kLowLow : kLow;
+ ne->value = 0;
+ ne->count = 0;
+ ne->thresh = 90;
+ ne->last_w = 0;
+ ne->last_h = 0;
+ if (width * height >= 1920 * 1080) {
+ ne->thresh = 200;
+ } else if (width * height >= 1280 * 720) {
+ ne->thresh = 140;
+ } else if (width * height >= 640 * 360) {
+ ne->thresh = 115;
+ }
+ ne->num_frames_estimate = 15;
+ ne->adapt_thresh = (3 * ne->thresh) >> 1;
+}
+
+static int enable_noise_estimation(AV1_COMP *const cpi) {
+ ResizePendingParams *const resize_pending_params =
+ &cpi->resize_pending_params;
+ const int resize_pending =
+ (resize_pending_params->width && resize_pending_params->height &&
+ (cpi->common.width != resize_pending_params->width ||
+ cpi->common.height != resize_pending_params->height));
+
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (cpi->common.seq_params.use_highbitdepth) return 0;
+#endif
+// Enable noise estimation if denoising is on.
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
+ cpi->common.width >= 320 && cpi->common.height >= 180)
+ return 1;
+#endif
+ // Only allow noise estimate under certain encoding mode.
+ // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original.
+ // Not enabled for SVC mode and screen_content_mode.
+ // Not enabled for low resolutions.
+ if (cpi->oxcf.pass == 0 && cpi->oxcf.rc_cfg.mode == AOM_CBR &&
+ cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 &&
+ resize_pending == 0 && !cpi->use_svc &&
+ cpi->oxcf.content != AOM_CONTENT_SCREEN &&
+ cpi->common.width * cpi->common.height >= 640 * 360)
+ return 1;
+ else
+ return 0;
+}
+
+#if CONFIG_AV1_TEMPORAL_DENOISING
+static void copy_frame(YV12_BUFFER_CONFIG *const dest,
+ const YV12_BUFFER_CONFIG *const src) {
+ const uint8_t *srcbuf = src->y_buffer;
+ uint8_t *destbuf = dest->y_buffer;
+
+ assert(dest->y_width == src->y_width);
+ assert(dest->y_height == src->y_height);
+
+ for (int r = 0; r < dest->y_height; ++r) {
+ memcpy(destbuf, srcbuf, dest->y_width);
+ destbuf += dest->y_stride;
+ srcbuf += src->y_stride;
+ }
+}
+#endif // CONFIG_AV1_TEMPORAL_DENOISING
+
+NOISE_LEVEL av1_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) {
+ int noise_level = kLowLow;
+ if (ne->value > (ne->thresh << 1)) {
+ noise_level = kHigh;
+ } else {
+ if (ne->value > ne->thresh)
+ noise_level = kMedium;
+ else if (ne->value > (ne->thresh >> 1))
+ noise_level = kLow;
+ else
+ noise_level = kLowLow;
+ }
+ return noise_level;
+}
+
+void av1_update_noise_estimate(AV1_COMP *const cpi) {
+ const AV1_COMMON *const cm = &cpi->common;
+ const CommonModeInfoParams *const mi_params = &cm->mi_params;
+
+ NOISE_ESTIMATE *const ne = &cpi->noise_estimate;
+ const int low_res = (cm->width <= 352 && cm->height <= 288);
+ // Estimate of noise level every frame_period frames.
+ int frame_period = 8;
+ int thresh_consec_zeromv = 6;
+ int frame_counter = cm->current_frame.frame_number;
+ // Estimate is between current source and last source.
+ YV12_BUFFER_CONFIG *last_source = cpi->last_source;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) {
+ last_source = &cpi->denoiser.last_source;
+ // Tune these thresholds for different resolutions when denoising is
+ // enabled.
+ if (cm->width > 640 && cm->width <= 1920) {
+ thresh_consec_zeromv = 2;
+ }
+ }
+#endif
+ ne->enabled = enable_noise_estimation(cpi);
+ if (cpi->svc.number_spatial_layers > 1)
+ frame_counter = cpi->svc.current_superframe;
+ if (!ne->enabled || frame_counter % frame_period != 0 ||
+ last_source == NULL ||
+ (cpi->svc.number_spatial_layers == 1 &&
+ (ne->last_w != cm->width || ne->last_h != cm->height))) {
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
+ copy_frame(&cpi->denoiser.last_source, cpi->source);
+#endif
+ if (last_source != NULL) {
+ ne->last_w = cm->width;
+ ne->last_h = cm->height;
+ }
+ return;
+ } else if (frame_counter > 60 && cpi->svc.num_encoded_top_layer > 1 &&
+ cpi->rc.frames_since_key > cpi->svc.number_spatial_layers &&
+ cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
+ cpi->rc.avg_frame_low_motion < (low_res ? 60 : 40)) {
+ // Force noise estimation to 0 and denoiser off if content has high motion.
+ ne->level = kLowLow;
+ ne->count = 0;
+ ne->num_frames_estimate = 10;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
+ cpi->svc.current_superframe > 1) {
+ av1_denoiser_set_noise_level(cpi, ne->level);
+ copy_frame(&cpi->denoiser.last_source, cpi->source);
+ }
+#endif
+ return;
+ } else {
+ unsigned int bin_size = 100;
+ unsigned int hist[MAX_VAR_HIST_BINS] = { 0 };
+ unsigned int hist_avg[MAX_VAR_HIST_BINS];
+ unsigned int max_bin = 0;
+ unsigned int max_bin_count = 0;
+ unsigned int bin_cnt;
+ int bsize = BLOCK_16X16;
+ // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
+ // been encoded as zero/small mv at least x consecutive frames, compute
+ // the variance to update estimate of noise in the source.
+ const uint8_t *src_y = cpi->source->y_buffer;
+ const int src_ystride = cpi->source->y_stride;
+ const uint8_t *last_src_y = last_source->y_buffer;
+ const int last_src_ystride = last_source->y_stride;
+ const uint8_t *src_u = cpi->source->u_buffer;
+ const uint8_t *src_v = cpi->source->v_buffer;
+ const int src_uvstride = cpi->source->uv_stride;
+ int mi_row, mi_col;
+ int num_low_motion = 0;
+ int frame_low_motion = 1;
+ for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row += 2) {
+ for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col += 2) {
+ int bl_index =
+ (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1);
+ if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv)
+ num_low_motion++;
+ }
+ }
+ if (num_low_motion <
+ (((3 * (mi_params->mi_rows * mi_params->mi_cols) >> 2)) >> 3))
+ frame_low_motion = 0;
+ for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row++) {
+ for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col++) {
+ // 16x16 blocks, 1/4 sample of frame.
+ if (mi_row % 8 == 0 && mi_col % 8 == 0 &&
+ mi_row < mi_params->mi_rows - 3 &&
+ mi_col < mi_params->mi_cols - 3) {
+ int bl_index =
+ (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1);
+ int bl_index1 = bl_index + 1;
+ int bl_index2 = bl_index + (mi_params->mi_cols >> 1);
+ int bl_index3 = bl_index2 + 1;
+ int consec_zeromv =
+ AOMMIN(cpi->consec_zero_mv[bl_index],
+ AOMMIN(cpi->consec_zero_mv[bl_index1],
+ AOMMIN(cpi->consec_zero_mv[bl_index2],
+ cpi->consec_zero_mv[bl_index3])));
+ // Only consider blocks that are likely steady background. i.e, have
+ // been encoded as zero/low motion x (= thresh_consec_zeromv) frames
+ // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all
+ // 4 sub-blocks for 16x16 block. And exclude this frame if
+ // high_source_sad is true (i.e., scene/content change).
+ if (frame_low_motion && consec_zeromv > thresh_consec_zeromv &&
+ !cpi->rc.high_source_sad) {
+ unsigned int sse;
+ // Compute variance between co-located blocks from current and
+ // last input frames.
+ unsigned int variance = cpi->fn_ptr[bsize].vf(
+ src_y, src_ystride, last_src_y, last_src_ystride, &sse);
+ unsigned int hist_index = variance / bin_size;
+ if (hist_index < MAX_VAR_HIST_BINS)
+ hist[hist_index]++;
+ else if (hist_index < 3 * (MAX_VAR_HIST_BINS >> 1))
+ hist[MAX_VAR_HIST_BINS - 1]++; // Account for the tail
+ }
+ }
+ src_y += 4;
+ last_src_y += 4;
+ src_u += 2;
+ src_v += 2;
+ }
+ src_y += (src_ystride << 2) - (mi_params->mi_cols << 2);
+ last_src_y += (last_src_ystride << 2) - (mi_params->mi_cols << 2);
+ src_u += (src_uvstride << 1) - (mi_params->mi_cols << 1);
+ src_v += (src_uvstride << 1) - (mi_params->mi_cols << 1);
+ }
+ ne->last_w = cm->width;
+ ne->last_h = cm->height;
+ // Adjust histogram to account for effect that histogram flattens
+ // and shifts to zero as scene darkens.
+ if (hist[0] > 10 && (hist[MAX_VAR_HIST_BINS - 1] > hist[0] >> 2)) {
+ hist[0] = 0;
+ hist[1] >>= 2;
+ hist[2] >>= 2;
+ hist[3] >>= 2;
+ hist[4] >>= 1;
+ hist[5] >>= 1;
+ hist[6] = 3 * hist[6] >> 1;
+ hist[MAX_VAR_HIST_BINS - 1] >>= 1;
+ }
+
+ // Average hist[] and find largest bin
+ for (bin_cnt = 0; bin_cnt < MAX_VAR_HIST_BINS; bin_cnt++) {
+ if (bin_cnt == 0)
+ hist_avg[bin_cnt] = (hist[0] + hist[1] + hist[2]) / 3;
+ else if (bin_cnt == MAX_VAR_HIST_BINS - 1)
+ hist_avg[bin_cnt] = hist[MAX_VAR_HIST_BINS - 1] >> 2;
+ else if (bin_cnt == MAX_VAR_HIST_BINS - 2)
+ hist_avg[bin_cnt] = (hist[bin_cnt - 1] + 2 * hist[bin_cnt] +
+ (hist[bin_cnt + 1] >> 1) + 2) >>
+ 2;
+ else
+ hist_avg[bin_cnt] =
+ (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + hist[bin_cnt + 1] + 2) >>
+ 2;
+
+ if (hist_avg[bin_cnt] > max_bin_count) {
+ max_bin_count = hist_avg[bin_cnt];
+ max_bin = bin_cnt;
+ }
+ }
+
+ // Scale by 40 to work with existing thresholds
+ ne->value = (int)((3 * ne->value + max_bin * 40) >> 2);
+ // Quickly increase VNR strength when the noise level increases suddenly.
+ if (ne->level < kMedium && ne->value > ne->adapt_thresh) {
+ ne->count = ne->num_frames_estimate;
+ } else {
+ ne->count++;
+ }
+ if (ne->count == ne->num_frames_estimate) {
+ // Reset counter and check noise level condition.
+ ne->num_frames_estimate = 30;
+ ne->count = 0;
+ ne->level = av1_noise_estimate_extract_level(ne);
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
+ av1_denoiser_set_noise_level(cpi, ne->level);
+#endif
+ }
+ }
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
+ copy_frame(&cpi->denoiser.last_source, cpi->source);
+#endif
+}
diff --git a/av1/encoder/av1_noise_estimate.h b/av1/encoder/av1_noise_estimate.h
new file mode 100644
index 0000000..8553066
--- /dev/null
+++ b/av1/encoder/av1_noise_estimate.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_AV1_NOISE_ESTIMATE_H_
+#define AOM_AV1_ENCODER_AV1_NOISE_ESTIMATE_H_
+
+#include "av1/encoder/block.h"
+#include "aom_scale/yv12config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MAX_VAR_HIST_BINS 20
+
+typedef enum noise_level { kLowLow, kLow, kMedium, kHigh } NOISE_LEVEL;
+
+typedef struct noise_estimate {
+ int enabled;
+ NOISE_LEVEL level;
+ int value;
+ int thresh;
+ int adapt_thresh;
+ int count;
+ int last_w;
+ int last_h;
+ int num_frames_estimate;
+} NOISE_ESTIMATE;
+
+struct AV1_COMP;
+
+void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height);
+
+NOISE_LEVEL av1_noise_estimate_extract_level(NOISE_ESTIMATE *const ne);
+
+void av1_update_noise_estimate(struct AV1_COMP *const cpi);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // AOM_AV1_ENCODER_AV1_NOISE_ESTIMATE_H_
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 04aaf46..694d6be 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6402,6 +6402,33 @@
}
}
+static void update_zeromv_cnt(const AV1_COMP *const cpi,
+ const MB_MODE_INFO *const mi, int mi_row,
+ int mi_col, BLOCK_SIZE bsize) {
+ const AV1_COMMON *const cm = &cpi->common;
+ MV mv = mi->mv[0].as_mv;
+ const int bw = mi_size_wide[bsize] >> 1;
+ const int bh = mi_size_high[bsize] >> 1;
+ const int xmis = AOMMIN((cm->mi_params.mi_cols - mi_col) >> 1, bw);
+ const int ymis = AOMMIN((cm->mi_params.mi_rows - mi_row) >> 1, bh);
+ const int block_index =
+ (mi_row >> 1) * (cm->mi_params.mi_cols >> 1) + (mi_col >> 1);
+ for (int y = 0; y < ymis; y++)
+ for (int x = 0; x < xmis; x++) {
+ // consec_zero_mv is in the scale of 8x8 blocks
+ const int map_offset = block_index + y * (cm->mi_params.mi_cols >> 1) + x;
+ if (mi->ref_frame[0] == LAST_FRAME && is_inter_block(mi) &&
+ mi->segment_id <= CR_SEGMENT_ID_BOOST2) {
+ if (abs(mv.row) < 10 && abs(mv.col) < 10) {
+ if (cpi->consec_zero_mv[map_offset] < 255)
+ cpi->consec_zero_mv[map_offset]++;
+ } else {
+ cpi->consec_zero_mv[map_offset] = 0;
+ }
+ }
+ }
+}
+
static AOM_INLINE void encode_superblock(const AV1_COMP *const cpi,
TileDataEnc *tile_data, ThreadData *td,
TokenExtra **t, RUN_TYPE dry_run,
@@ -6581,4 +6608,13 @@
if (is_inter_block(mbmi) && !xd->is_chroma_ref && is_cfl_allowed(xd)) {
cfl_store_block(xd, mbmi->sb_type, mbmi->tx_size);
}
+ if (!dry_run) {
+ if (cpi->oxcf.pass == 0 && cpi->svc.temporal_layer_id == 0 &&
+ cpi->sf.rt_sf.use_temporal_noise_estimate &&
+ (!cpi->use_svc ||
+ (cpi->use_svc &&
+ !cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame &&
+ cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1)))
+ update_zeromv_cnt(cpi, mbmi, mi_row, mi_col, bsize);
+ }
}
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index a7984e5..98e70c8 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -875,6 +875,8 @@
resize_pending_params->height = 0;
init_buffer_indices(&cpi->force_intpel_info, cm->remapped_ref_idx);
+
+ av1_noise_estimate_init(&cpi->noise_estimate, cm->width, cm->height);
}
void av1_change_config(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf) {
@@ -1289,6 +1291,10 @@
av1_set_speed_features_framesize_independent(cpi, oxcf->speed);
av1_set_speed_features_framesize_dependent(cpi, oxcf->speed);
+ CHECK_MEM_ERROR(cm, cpi->consec_zero_mv,
+ aom_calloc((mi_params->mi_rows * mi_params->mi_cols) >> 2,
+ sizeof(*cpi->consec_zero_mv)));
+
{
const int bsize = BLOCK_16X16;
const int w = mi_size_wide[bsize];
@@ -2381,6 +2387,8 @@
// Recalculate 'all_lossless' in case super-resolution was (un)selected.
cm->features.all_lossless =
cm->features.coded_lossless && !av1_superres_scaled(cm);
+
+ av1_noise_estimate_init(&cpi->noise_estimate, cm->width, cm->height);
}
set_mv_search_params(cpi);
@@ -2856,6 +2864,13 @@
AV1_COMMON *const cm = &cpi->common;
const QuantizationCfg *const q_cfg = &cpi->oxcf.q_cfg;
SVC *const svc = &cpi->svc;
+ ResizePendingParams *const resize_pending_params =
+ &cpi->resize_pending_params;
+ const int resize_pending =
+ (resize_pending_params->width && resize_pending_params->height &&
+ (cpi->common.width != resize_pending_params->width ||
+ cpi->common.height != resize_pending_params->height));
+
int top_index = 0, bottom_index = 0, q = 0;
YV12_BUFFER_CONFIG *unscaled = cpi->unscaled_source;
InterpFilter filter_scaler =
@@ -2887,12 +2902,22 @@
cpi->source = av1_scale_if_required(cm, unscaled, &cpi->scaled_source,
filter_scaler, phase_scaler);
+ if (frame_is_intra_only(cm) || resize_pending != 0) {
+ memset(cpi->consec_zero_mv, 0,
+ ((cm->mi_params.mi_rows * cm->mi_params.mi_cols) >> 2) *
+ sizeof(*cpi->consec_zero_mv));
+ }
+
if (cpi->unscaled_last_source != NULL) {
cpi->last_source = av1_scale_if_required(cm, cpi->unscaled_last_source,
&cpi->scaled_last_source,
filter_scaler, phase_scaler);
}
+ if (cpi->sf.rt_sf.use_temporal_noise_estimate) {
+ av1_update_noise_estimate(cpi);
+ }
+
// For SVC the inter-layer/spatial prediction is not done for newmv
// (zero_mode is forced), and since the scaled references are only
// use for newmv search, we can avoid scaling here.
@@ -4060,6 +4085,9 @@
}
}
+ if (cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1)
+ cpi->svc.num_encoded_top_layer++;
+
#if DUMP_RECON_FRAMES == 1
// NOTE(zoeliu): For debug - Output the filtered reconstructed video.
dump_filtered_recon_frames(cpi);
@@ -4129,6 +4157,15 @@
av1_rc_postencode_update(cpi, *size);
+ if (oxcf->pass == 0 && !frame_is_intra_only(cm) &&
+ cpi->sf.rt_sf.use_temporal_noise_estimate &&
+ (!cpi->use_svc ||
+ (cpi->use_svc &&
+ !cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame &&
+ cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1))) {
+ av1_compute_frame_low_motion(cpi);
+ }
+
// Clear the one shot update flags for segmentation map and mode/ref loop
// filter deltas.
cm->seg.update_map = 0;
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index f9b4a23..d8f5a0a 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -46,6 +46,7 @@
#include "av1/encoder/svc_layercontext.h"
#include "av1/encoder/tokenize.h"
#include "av1/encoder/tpl_model.h"
+#include "av1/encoder/av1_noise_estimate.h"
#if CONFIG_INTERNAL_STATS
#include "aom_dsp/ssim.h"
@@ -2312,6 +2313,17 @@
* First pass related data.
*/
FirstPassData firstpass_data;
+
+ /*!
+ * Temporal Noise Estimate
+ */
+ NOISE_ESTIMATE noise_estimate;
+
+ /*!
+ * Count on how many consecutive times a block uses small/zeromv for encoding
+ * in a scale of 8x8 block.
+ */
+ uint8_t *consec_zero_mv;
} AV1_COMP;
/*!\cond */
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h
index 985313b..57331e0 100644
--- a/av1/encoder/encoder_alloc.h
+++ b/av1/encoder/encoder_alloc.h
@@ -304,6 +304,11 @@
}
if (cpi->use_svc) av1_free_svc_cyclic_refresh(cpi);
+
+ if (cpi->consec_zero_mv) {
+ aom_free(cpi->consec_zero_mv);
+ cpi->consec_zero_mv = NULL;
+ }
}
static AOM_INLINE void variance_partition_alloc(AV1_COMP *cpi) {
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 03b94dd..de0bf40 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -338,6 +338,7 @@
rc->max_gf_interval = av1_rc_get_default_max_gf_interval(
oxcf->init_framerate, rc->min_gf_interval);
rc->baseline_gf_interval = (rc->min_gf_interval + rc->max_gf_interval) / 2;
+ rc->avg_frame_low_motion = 0;
}
int av1_rc_drop_frame(AV1_COMP *cpi) {
@@ -2427,3 +2428,36 @@
return 0;
}
}
+
+void av1_compute_frame_low_motion(AV1_COMP *const cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ const CommonModeInfoParams *const mi_params = &cm->mi_params;
+ SVC *const svc = &cpi->svc;
+ MB_MODE_INFO **mi = mi_params->mi_grid_base;
+ RATE_CONTROL *const rc = &cpi->rc;
+ const int rows = mi_params->mi_rows, cols = mi_params->mi_cols;
+ int cnt_zeromv = 0;
+ for (int mi_row = 0; mi_row < rows; mi_row++) {
+ for (int mi_col = 0; mi_col < cols; mi_col++) {
+ if (mi[0]->ref_frame[0] == LAST_FRAME &&
+ abs(mi[0]->mv[0].as_mv.row) < 16 && abs(mi[0]->mv[0].as_mv.col) < 16)
+ cnt_zeromv++;
+ mi++;
+ }
+ mi += mi_params->mi_stride - cols;
+ }
+ cnt_zeromv = 100 * cnt_zeromv / (rows * cols);
+ rc->avg_frame_low_motion = (3 * rc->avg_frame_low_motion + cnt_zeromv) >> 2;
+
+ // For SVC: set avg_frame_low_motion (only computed on top spatial layer)
+ // to all lower spatial layers.
+ if (cpi->use_svc && svc->spatial_layer_id == svc->number_spatial_layers - 1) {
+ for (int i = 0; i < svc->number_spatial_layers - 1; ++i) {
+ const int layer = LAYER_IDS_TO_IDX(i, svc->temporal_layer_id,
+ svc->number_temporal_layers);
+ LAYER_CONTEXT *const lc = &svc->layer_context[layer];
+ RATE_CONTROL *const lrc = &lc->rc;
+ lrc->avg_frame_low_motion = rc->avg_frame_low_motion;
+ }
+ }
+}
diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h
index 311174d..f32101d 100644
--- a/av1/encoder/ratectrl.h
+++ b/av1/encoder/ratectrl.h
@@ -192,6 +192,8 @@
int next_is_fwd_key;
int enable_scenecut_detection;
int use_arf_in_this_kf_group;
+ // Track amount of low motion in scene
+ int avg_frame_low_motion;
} RATE_CONTROL;
struct AV1_COMP;
@@ -341,6 +343,8 @@
int av1_encodedframe_overshoot(struct AV1_COMP *cpi, int *q);
+void av1_compute_frame_low_motion(struct AV1_COMP *const cpi);
+
/*!\endcond */
#ifdef __cplusplus
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index d3ed7ba..7a57140 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -862,6 +862,9 @@
sf->rt_sf.overshoot_detection_cbr = FAST_DETECTION_MAXQ;
sf->rt_sf.check_scene_detection = 1;
}
+ // Keeping this off for now as some clips show ~6% BDRate regression with
+ // moderate speed-up (~20%)
+ sf->rt_sf.use_temporal_noise_estimate = 0;
}
if (speed >= 6) {
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 177e6db..ef833bc 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -984,6 +984,9 @@
// Only checks intra DCPRED mode in nonrd_pick_inter_mode
int nonrd_intra_dc_only;
+
+ // uses results of temporal noise estimate
+ int use_temporal_noise_estimate;
} REAL_TIME_SPEED_FEATURES;
typedef struct SPEED_FEATURES {
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c
index b57b85a..ba4c2e8 100644
--- a/av1/encoder/svc_layercontext.c
+++ b/av1/encoder/svc_layercontext.c
@@ -29,6 +29,7 @@
svc->base_framerate = 30.0;
svc->current_superframe = 0;
svc->force_zero_mode_spatial_ref = 1;
+ svc->num_encoded_top_layer = 0;
for (int sl = 0; sl < svc->number_spatial_layers; ++sl) {
for (int tl = 0; tl < svc->number_temporal_layers; ++tl) {
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h
index 07d8148..74c9dc9 100644
--- a/av1/encoder/svc_layercontext.h
+++ b/av1/encoder/svc_layercontext.h
@@ -64,6 +64,7 @@
int skip_nonzeromv_gf;
int spatial_layer_fb[REF_FRAMES];
int temporal_layer_fb[REF_FRAMES];
+ int num_encoded_top_layer;
// Layer context used for rate control in CBR mode.
LAYER_CONTEXT layer_context[AOM_MAX_LAYERS];
// EIGHTTAP_SMOOTH or BILINEAR
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 6fc0c38..56abf99 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -358,6 +358,17 @@
thresholds[3] = threshold_base >> 2;
thresholds[4] = threshold_base << 2;
} else {
+ if (cpi->noise_estimate.enabled && cm->width >= 640 && cm->height >= 480) {
+ NOISE_LEVEL noise_level =
+ av1_noise_estimate_extract_level(&cpi->noise_estimate);
+ if (noise_level == kHigh)
+ threshold_base = 3 * threshold_base;
+ else if (noise_level == kMedium)
+ threshold_base = threshold_base << 1;
+ else if (noise_level < kLow)
+ threshold_base = (7 * threshold_base) >> 3;
+ }
+
// Increase base variance threshold based on content_state/sum_diff level.
threshold_base = scale_part_thresh_sumdiff(
threshold_base, cpi->oxcf.speed, cm->width, cm->height, content_state);
@@ -808,6 +819,7 @@
VP16x16 *vt2 = NULL;
unsigned char force_split[85];
int avg_32x32;
+ int avg_64x64;
int max_var_32x32[4];
int min_var_32x32[4];
int var_32x32;
@@ -840,6 +852,7 @@
// Ref frame used in partitioning.
MV_REFERENCE_FRAME ref_frame_partition = LAST_FRAME;
+ NOISE_LEVEL noise_level = kLow;
CHECK_MEM_ERROR(cm, vt, aom_malloc(sizeof(*vt)));
@@ -909,6 +922,10 @@
thresholds, s, sp, d, dp);
// Fill the rest of the variance tree by summing split partition values.
+ if (cpi->noise_estimate.enabled)
+ noise_level = av1_noise_estimate_extract_level(&cpi->noise_estimate);
+
+ avg_64x64 = 0;
for (m = 0; m < num_64x64_blocks; ++m) {
avg_32x32 = 0;
max_var_32x32[m] = 0;
@@ -955,7 +972,10 @@
force_split[5 + m2 + i] = 1;
force_split[m + 1] = 1;
force_split[0] = 1;
- } else if (!is_key_frame && cm->height <= 360 &&
+ } else if (!is_key_frame &&
+ (!cpi->noise_estimate.enabled ||
+ (cpi->noise_estimate.enabled && noise_level < kLow)) &&
+ cm->height <= 360 &&
(maxvar_16x16[m][i] - minvar_16x16[m][i]) >
(thresholds[2] >> 1) &&
maxvar_16x16[m][i] > thresholds[2]) {
@@ -980,6 +1000,7 @@
(max_var_32x32[m] - min_var_32x32[m]) > 3 * (thresholds[1] >> 3) &&
max_var_32x32[m] > thresholds[1] >> 1)
force_split[1 + m] = 1;
+ avg_64x64 += var_64x64;
}
if (is_small_sb) force_split[0] = 1;
}
@@ -988,6 +1009,12 @@
fill_variance_tree(vt, BLOCK_128X128);
get_variance(&vt->part_variances.none);
if (!is_key_frame &&
+ (!cpi->noise_estimate.enabled ||
+ (cpi->noise_estimate.enabled && noise_level >= kMedium)) &&
+ vt->part_variances.none.variance > (9 * avg_64x64) >> 5)
+ force_split[0] = 1;
+
+ if (!is_key_frame &&
(max_var_64x64 - min_var_64x64) > 3 * (thresholds[0] >> 3) &&
max_var_64x64 > thresholds[0] >> 1)
force_split[0] = 1;
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake
index b75910b..37c779d 100644
--- a/build/cmake/aom_config_defaults.cmake
+++ b/build/cmake/aom_config_defaults.cmake
@@ -132,6 +132,8 @@
"Build for RTC-only to reduce binary size.")
set_aom_config_var(CONFIG_AV1_HIGHBITDEPTH 1
"Build with high bitdepth support.")
+set_aom_config_var(CONFIG_AV1_TEMPORAL_DENOISING 0
+ "Build with temporal denoising support.")
set_aom_config_var(CONFIG_NN_V2 0 "Fully-connected neural nets ver.2.")
set_aom_config_var(CONFIG_SUPERRES_IN_RECODE 1
"Enable encoding both full-res and superres in recode loop"
