AV1 RT: Implement AV1 temporal denoising
Based on VP9 implementation. To enable: compile with
CONFIG_AV1_TEMPORAL_DENOISING flag.
Current limitations:
- SVC implementation is not complete
- NEON and SSE optimization to follow
- No skin detection
- No impact of Variance based bartition
Change-Id: Id70fe4022689c12938159fd4569d5091115f9052
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 718254b..eedc5d8 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -271,6 +271,12 @@
"${AOM_ROOT}/av1/encoder/optical_flow.h")
endif()
+if(CONFIG_AV1_TEMPORAL_DENOISING)
+ list(APPEND AOM_AV1_ENCODER_SOURCES
+ "${AOM_ROOT}/av1/encoder/av1_temporal_denoiser.c"
+ "${AOM_ROOT}/av1/encoder/av1_temporal_denoiser.h")
+endif()
+
list(APPEND AOM_AV1_COMMON_INTRIN_SSE2
"${AOM_ROOT}/av1/common/cdef_block_sse2.c"
"${AOM_ROOT}/av1/common/x86/cfl_sse2.c"
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 3ea3b8f..17250d2 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -962,6 +962,9 @@
oxcf->noise_block_size = extra_cfg->noise_block_size;
#endif
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ oxcf->noise_sensitivity = extra_cfg->noise_sensitivity;
+#endif
// Set Tile related configuration.
tile_cfg->num_tile_groups = extra_cfg->num_tg;
// In large-scale tile encoding mode, num_tile_groups is always 1.
diff --git a/av1/encoder/av1_noise_estimate.c b/av1/encoder/av1_noise_estimate.c
index 0e6ee15..dbc86c5 100644
--- a/av1/encoder/av1_noise_estimate.c
+++ b/av1/encoder/av1_noise_estimate.c
@@ -20,6 +20,9 @@
#include "av1/encoder/context_tree.h"
#include "av1/encoder/av1_noise_estimate.h"
#include "av1/encoder/encoder.h"
+#if CONFIG_AV1_TEMPORAL_DENOISING
+#include "av1/encoder/av1_temporal_denoiser.h"
+#endif
#if CONFIG_AV1_TEMPORAL_DENOISING
// For SVC: only do noise estimation on top spatial layer.
diff --git a/av1/encoder/av1_temporal_denoiser.c b/av1/encoder/av1_temporal_denoiser.c
new file mode 100644
index 0000000..f8cdc69
--- /dev/null
+++ b/av1/encoder/av1_temporal_denoiser.c
@@ -0,0 +1,802 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <assert.h>
+#include <limits.h>
+#include <math.h>
+
+#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_scale/yv12config.h"
+#include "aom/aom_integer.h"
+#include "av1/common/reconinter.h"
+#include "av1/encoder/reconinter_enc.h"
+#include "av1/encoder/context_tree.h"
+#include "av1/encoder/av1_temporal_denoiser.h"
+#include "av1/encoder/encoder.h"
+
+#ifdef OUTPUT_YUV_DENOISED
+static void make_grayscale(YV12_BUFFER_CONFIG *yuv);
+#endif
+
+static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) {
+ (void)bs;
+ return 3 + (increase_denoising ? 1 : 0);
+}
+
+static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) {
+ (void)bs;
+ (void)increase_denoising;
+ return 4;
+}
+
+static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) {
+ (void)bs;
+ (void)increase_denoising;
+ return 625;
+}
+
+static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) {
+ return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 80 : 40);
+}
+
+static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising,
+ int motion_magnitude) {
+ if (motion_magnitude > noise_motion_thresh(bs, increase_denoising)) {
+ if (increase_denoising)
+ return (1 << num_pels_log2_lookup[bs]) << 2;
+ else
+ return 0;
+ } else {
+ return (1 << num_pels_log2_lookup[bs]) << 4;
+ }
+}
+
+static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) {
+ return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2);
+}
+
+// TODO(kyslov): If increase_denoising is enabled in the future,
+// we might need to update the code for calculating 'total_adj' in
+// case the C code is not bit-exact with corresponding sse2 code.
+int av1_denoiser_filter_c(const uint8_t *sig, int sig_stride,
+ const uint8_t *mc_avg, int mc_avg_stride,
+ uint8_t *avg, int avg_stride, int increase_denoising,
+ BLOCK_SIZE bs, int motion_magnitude) {
+ int r, c;
+ const uint8_t *sig_start = sig;
+ const uint8_t *mc_avg_start = mc_avg;
+ uint8_t *avg_start = avg;
+ int diff, adj, absdiff, delta;
+ int adj_val[] = { 3, 4, 6 };
+ int total_adj = 0;
+ int shift_inc = 1;
+
+ // If motion_magnitude is small, making the denoiser more aggressive by
+ // increasing the adjustment for each level. Add another increment for
+ // blocks that are labeled for increase denoising.
+ if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) {
+ if (increase_denoising) {
+ shift_inc = 2;
+ }
+ adj_val[0] += shift_inc;
+ adj_val[1] += shift_inc;
+ adj_val[2] += shift_inc;
+ }
+
+ // First attempt to apply a strong temporal denoising filter.
+ for (r = 0; r < block_size_high[bs]; ++r) {
+ for (c = 0; c < block_size_wide[bs]; ++c) {
+ diff = mc_avg[c] - sig[c];
+ absdiff = abs(diff);
+
+ if (absdiff <= absdiff_thresh(bs, increase_denoising)) {
+ avg[c] = mc_avg[c];
+ total_adj += diff;
+ } else {
+ switch (absdiff) {
+ case 4:
+ case 5:
+ case 6:
+ case 7: adj = adj_val[0]; break;
+ case 8:
+ case 9:
+ case 10:
+ case 11:
+ case 12:
+ case 13:
+ case 14:
+ case 15: adj = adj_val[1]; break;
+ default: adj = adj_val[2];
+ }
+ if (diff > 0) {
+ avg[c] = AOMMIN(UINT8_MAX, sig[c] + adj);
+ total_adj += adj;
+ } else {
+ avg[c] = AOMMAX(0, sig[c] - adj);
+ total_adj -= adj;
+ }
+ }
+ }
+ sig += sig_stride;
+ avg += avg_stride;
+ mc_avg += mc_avg_stride;
+ }
+
+ // If the strong filter did not modify the signal too much, we're all set.
+ if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) {
+ return FILTER_BLOCK;
+ }
+
+ // Otherwise, we try to dampen the filter if the delta is not too high.
+ delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising)) >>
+ num_pels_log2_lookup[bs]) +
+ 1;
+
+ if (delta >= delta_thresh(bs, increase_denoising)) {
+ return COPY_BLOCK;
+ }
+
+ mc_avg = mc_avg_start;
+ avg = avg_start;
+ sig = sig_start;
+ for (r = 0; r < block_size_high[bs]; ++r) {
+ for (c = 0; c < block_size_wide[bs]; ++c) {
+ diff = mc_avg[c] - sig[c];
+ adj = abs(diff);
+ if (adj > delta) {
+ adj = delta;
+ }
+ if (diff > 0) {
+ // Diff positive means we made positive adjustment above
+ // (in first try/attempt), so now make negative adjustment to bring
+ // denoised signal down.
+ avg[c] = AOMMAX(0, avg[c] - adj);
+ total_adj -= adj;
+ } else {
+ // Diff negative means we made negative adjustment above
+ // (in first try/attempt), so now make positive adjustment to bring
+ // denoised signal up.
+ avg[c] = AOMMIN(UINT8_MAX, avg[c] + adj);
+ total_adj += adj;
+ }
+ }
+ sig += sig_stride;
+ avg += avg_stride;
+ mc_avg += mc_avg_stride;
+ }
+
+ // We can use the filter if it has been sufficiently dampened
+ if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) {
+ return FILTER_BLOCK;
+ }
+ return COPY_BLOCK;
+}
+
+static uint8_t *block_start(uint8_t *framebuf, int stride, int mi_row,
+ int mi_col) {
+ return framebuf + (stride * mi_row << 2) + (mi_col << 2);
+}
+
+static AV1_DENOISER_DECISION perform_motion_compensation(
+ AV1_COMMON *const cm, AV1_DENOISER *denoiser, MACROBLOCK *mb, BLOCK_SIZE bs,
+ int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx,
+ int motion_magnitude, int *zeromv_filter, int num_spatial_layers, int width,
+ int lst_fb_idx, int gld_fb_idx, int use_svc, int spatial_layer,
+ int use_gf_temporal_ref) {
+ const int sse_diff = (ctx->newmv_sse == UINT_MAX)
+ ? 0
+ : ((int)ctx->zeromv_sse - (int)ctx->newmv_sse);
+ int frame;
+ int denoise_layer_idx = 0;
+ MACROBLOCKD *filter_mbd = &mb->e_mbd;
+ MB_MODE_INFO *mi = filter_mbd->mi[0];
+ MB_MODE_INFO saved_mi;
+ int i;
+ struct buf_2d saved_dst[MAX_MB_PLANE];
+ struct buf_2d saved_pre[MAX_MB_PLANE];
+ // const RefBuffer *saved_block_refs[2];
+ MV_REFERENCE_FRAME saved_frame;
+
+ frame = ctx->best_reference_frame;
+
+ saved_mi = *mi;
+
+ // Avoid denoising small blocks. When noise > kDenLow or frame width > 480,
+ // denoise 16x16 blocks.
+ if (bs == BLOCK_8X8 || bs == BLOCK_8X16 || bs == BLOCK_16X8 ||
+ (bs == BLOCK_16X16 && width > 480 &&
+ denoiser->denoising_level <= kDenLow))
+ return COPY_BLOCK;
+
+ // If the best reference frame uses inter-prediction and there is enough of a
+ // difference in sum-squared-error, use it.
+ if (frame != INTRA_FRAME && frame != ALTREF_FRAME && frame != GOLDEN_FRAME &&
+ sse_diff > sse_diff_thresh(bs, increase_denoising, motion_magnitude)) {
+ mi->ref_frame[0] = ctx->best_reference_frame;
+ mi->mode = ctx->best_sse_inter_mode;
+ mi->mv[0] = ctx->best_sse_mv;
+ } else {
+ // Otherwise, use the zero reference frame.
+ frame = ctx->best_zeromv_reference_frame;
+ ctx->newmv_sse = ctx->zeromv_sse;
+ // Bias to last reference.
+ if ((num_spatial_layers > 1 && !use_gf_temporal_ref) ||
+ frame == ALTREF_FRAME ||
+ (frame == GOLDEN_FRAME && use_gf_temporal_ref) ||
+ (frame != LAST_FRAME &&
+ ((ctx->zeromv_lastref_sse<(5 * ctx->zeromv_sse)>> 2) ||
+ denoiser->denoising_level >= kDenHigh))) {
+ frame = LAST_FRAME;
+ ctx->newmv_sse = ctx->zeromv_lastref_sse;
+ }
+ mi->ref_frame[0] = frame;
+ mi->mode = GLOBALMV;
+ mi->mv[0].as_int = 0;
+ ctx->best_sse_inter_mode = GLOBALMV;
+ ctx->best_sse_mv.as_int = 0;
+ *zeromv_filter = 1;
+ if (denoiser->denoising_level > kDenMedium) {
+ motion_magnitude = 0;
+ }
+ }
+
+ saved_frame = frame;
+ // When using SVC, we need to map REF_FRAME to the frame buffer index.
+ if (use_svc) {
+ if (frame == LAST_FRAME)
+ frame = lst_fb_idx + 1;
+ else if (frame == GOLDEN_FRAME)
+ frame = gld_fb_idx + 1;
+ // Shift for the second spatial layer.
+ if (num_spatial_layers - spatial_layer == 2)
+ frame = frame + denoiser->num_ref_frames;
+ denoise_layer_idx = num_spatial_layers - spatial_layer - 1;
+ }
+
+ // Force copy (no denoise, copy source in denoised buffer) if
+ // running_avg_y[frame] is NULL.
+ if (denoiser->running_avg_y[frame].buffer_alloc == NULL) {
+ // Restore everything to its original state
+ *mi = saved_mi;
+ return COPY_BLOCK;
+ }
+
+ if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
+ // Restore everything to its original state
+ *mi = saved_mi;
+ return COPY_BLOCK;
+ }
+ if (motion_magnitude > (noise_motion_thresh(bs, increase_denoising) << 3)) {
+ // Restore everything to its original state
+ *mi = saved_mi;
+ return COPY_BLOCK;
+ }
+
+ // We will restore these after motion compensation.
+ for (i = 0; i < MAX_MB_PLANE; ++i) {
+ saved_pre[i] = filter_mbd->plane[i].pre[0];
+ saved_dst[i] = filter_mbd->plane[i].dst;
+ }
+
+ // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser
+ // struct.
+ set_ref_ptrs(cm, filter_mbd, saved_frame, NONE);
+ av1_setup_pre_planes(filter_mbd, 0, &(denoiser->running_avg_y[frame]), mi_row,
+ mi_col, filter_mbd->block_ref_scale_factors[0], 1);
+ av1_setup_dst_planes(filter_mbd->plane, bs,
+ &(denoiser->mc_running_avg_y[denoise_layer_idx]), mi_row,
+ mi_col, 0, 1);
+
+ av1_enc_build_inter_predictor_y(filter_mbd, mi_row, mi_col);
+
+ // Restore everything to its original state
+ *mi = saved_mi;
+ for (i = 0; i < MAX_MB_PLANE; ++i) {
+ filter_mbd->plane[i].pre[0] = saved_pre[i];
+ filter_mbd->plane[i].dst = saved_dst[i];
+ }
+
+ return FILTER_BLOCK;
+}
+
+void av1_denoiser_denoise(AV1_COMP *cpi, MACROBLOCK *mb, int mi_row, int mi_col,
+ BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
+ AV1_DENOISER_DECISION *denoiser_decision,
+ int use_gf_temporal_ref) {
+ int mv_col, mv_row;
+ int motion_magnitude = 0;
+ int zeromv_filter = 0;
+ AV1_DENOISER *denoiser = &cpi->denoiser;
+ AV1_DENOISER_DECISION decision = COPY_BLOCK;
+
+ const int shift =
+ cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id == 2
+ ? denoiser->num_ref_frames
+ : 0;
+ YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME + shift];
+ const int denoise_layer_index =
+ cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id - 1;
+ YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y[denoise_layer_index];
+ uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
+
+ uint8_t *mc_avg_start =
+ block_start(mc_avg.y_buffer, mc_avg.y_stride, mi_row, mi_col);
+ struct buf_2d src = mb->plane[0].src;
+ int increase_denoising = 0;
+ int last_is_reference = cpi->ref_frame_flags & AOM_LAST_FLAG;
+ mv_col = ctx->best_sse_mv.as_mv.col;
+ mv_row = ctx->best_sse_mv.as_mv.row;
+ motion_magnitude = mv_row * mv_row + mv_col * mv_col;
+
+ if (denoiser->denoising_level == kDenHigh) increase_denoising = 1;
+
+ // Copy block if LAST_FRAME is not a reference.
+ // Last doesn't always exist when SVC layers are dynamically changed, e.g. top
+ // spatial layer doesn't have last reference when it's brought up for the
+ // first time on the fly.
+ if (last_is_reference && denoiser->denoising_level >= kDenLow &&
+ !ctx->sb_skip_denoising)
+ decision = perform_motion_compensation(
+ &cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx,
+ motion_magnitude, &zeromv_filter, cpi->svc.number_spatial_layers,
+ cpi->source->y_width, cpi->svc.ref_idx[0], cpi->svc.ref_idx[3],
+ cpi->use_svc, cpi->svc.spatial_layer_id, use_gf_temporal_ref);
+
+ if (decision == FILTER_BLOCK) {
+ decision = av1_denoiser_filter_c(src.buf, src.stride, mc_avg_start,
+ mc_avg.y_stride, avg_start, avg.y_stride,
+ increase_denoising, bs, motion_magnitude);
+ }
+
+ if (decision == FILTER_BLOCK) {
+ aom_convolve_copy(avg_start, avg.y_stride, src.buf, src.stride,
+ block_size_wide[bs], block_size_high[bs]);
+ } else { // COPY_BLOCK
+ aom_convolve_copy(src.buf, src.stride, avg_start, avg.y_stride,
+ block_size_wide[bs], block_size_high[bs]);
+ }
+ *denoiser_decision = decision;
+ if (decision == FILTER_BLOCK && zeromv_filter == 1)
+ *denoiser_decision = FILTER_ZEROMV_BLOCK;
+}
+
+static void copy_frame(YV12_BUFFER_CONFIG *const dest,
+ const YV12_BUFFER_CONFIG *const src) {
+ int r;
+ const uint8_t *srcbuf = src->y_buffer;
+ uint8_t *destbuf = dest->y_buffer;
+
+ assert(dest->y_width == src->y_width);
+ assert(dest->y_height == src->y_height);
+
+ for (r = 0; r < dest->y_height; ++r) {
+ memcpy(destbuf, srcbuf, dest->y_width);
+ destbuf += dest->y_stride;
+ srcbuf += src->y_stride;
+ }
+}
+
+static void swap_frame_buffer(YV12_BUFFER_CONFIG *const dest,
+ YV12_BUFFER_CONFIG *const src) {
+ uint8_t *tmp_buf = dest->y_buffer;
+ assert(dest->y_width == src->y_width);
+ assert(dest->y_height == src->y_height);
+ dest->y_buffer = src->y_buffer;
+ src->y_buffer = tmp_buf;
+}
+
+void av1_denoiser_update_frame_info(
+ AV1_DENOISER *denoiser, YV12_BUFFER_CONFIG src, struct SVC *svc,
+ FRAME_TYPE frame_type, int refresh_alt_ref_frame, int refresh_golden_frame,
+ int refresh_last_frame, int alt_fb_idx, int gld_fb_idx, int lst_fb_idx,
+ int resized, int svc_refresh_denoiser_buffers, int second_spatial_layer) {
+ const int shift = second_spatial_layer ? denoiser->num_ref_frames : 0;
+ // Copy source into denoised reference buffers on KEY_FRAME or
+ // if the just encoded frame was resized. For SVC, copy source if the base
+ // spatial layer was key frame.
+ if (frame_type == KEY_FRAME || resized != 0 || denoiser->reset ||
+ svc_refresh_denoiser_buffers) {
+ int i;
+ // Start at 1 so as not to overwrite the INTRA_FRAME
+ for (i = 1; i < denoiser->num_ref_frames; ++i) {
+ if (denoiser->running_avg_y[i + shift].buffer_alloc != NULL)
+ copy_frame(&denoiser->running_avg_y[i + shift], &src);
+ }
+ denoiser->reset = 0;
+ return;
+ }
+
+ if (svc->external_ref_frame_config) {
+ int i;
+ for (i = 0; i < REF_FRAMES; i++) {
+ if (svc->refresh[svc->spatial_layer_id] & (1 << i))
+ copy_frame(&denoiser->running_avg_y[i + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ } else {
+ // If more than one refresh occurs, must copy frame buffer.
+ if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) >
+ 1) {
+ if (refresh_alt_ref_frame) {
+ copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ if (refresh_golden_frame) {
+ copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ if (refresh_last_frame) {
+ copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ } else {
+ if (refresh_alt_ref_frame) {
+ swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ if (refresh_golden_frame) {
+ swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ if (refresh_last_frame) {
+ swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1 + shift],
+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
+ }
+ }
+ }
+}
+
+void av1_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {
+ ctx->zeromv_sse = INT64_MAX;
+ ctx->newmv_sse = INT64_MAX;
+ ctx->zeromv_lastref_sse = INT64_MAX;
+ ctx->best_sse_mv.as_int = 0;
+}
+
+void av1_denoiser_update_frame_stats(MB_MODE_INFO *mi, int64_t sse,
+ PREDICTION_MODE mode,
+ PICK_MODE_CONTEXT *ctx) {
+ if (mi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
+ ctx->zeromv_sse = sse;
+ ctx->best_zeromv_reference_frame = mi->ref_frame[0];
+ if (mi->ref_frame[0] == LAST_FRAME) ctx->zeromv_lastref_sse = sse;
+ }
+
+ if (mi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
+ ctx->newmv_sse = sse;
+ ctx->best_sse_inter_mode = mode;
+ ctx->best_sse_mv = mi->mv[0];
+ ctx->best_reference_frame = mi->ref_frame[0];
+ }
+}
+
+static int av1_denoiser_realloc_svc_helper(AV1_COMMON *cm,
+ AV1_DENOISER *denoiser, int fb_idx) {
+ int fail = 0;
+ if (denoiser->running_avg_y[fb_idx].buffer_alloc == NULL) {
+ fail = aom_alloc_frame_buffer(
+ &denoiser->running_avg_y[fb_idx], cm->width, cm->height,
+ cm->seq_params.subsampling_x, cm->seq_params.subsampling_y,
+ cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS,
+ cm->features.byte_alignment);
+ if (fail) {
+ av1_denoiser_free(denoiser);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+int av1_denoiser_realloc_svc(AV1_COMMON *cm, AV1_DENOISER *denoiser,
+ struct SVC *svc, int svc_buf_shift,
+ int refresh_alt, int refresh_gld, int refresh_lst,
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) {
+ int fail = 0;
+ if (svc->external_ref_frame_config) {
+ int i;
+ for (i = 0; i < REF_FRAMES; i++) {
+ if (cm->current_frame.frame_type == KEY_FRAME ||
+ svc->refresh[svc->spatial_layer_id] & (1 << i)) {
+ fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
+ i + 1 + svc_buf_shift);
+ }
+ }
+ } else {
+ if (refresh_alt) {
+ // Increase the frame buffer index by 1 to map it to the buffer index in
+ // the denoiser.
+ fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
+ alt_fb_idx + 1 + svc_buf_shift);
+ if (fail) return 1;
+ }
+ if (refresh_gld) {
+ fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
+ gld_fb_idx + 1 + svc_buf_shift);
+ if (fail) return 1;
+ }
+ if (refresh_lst) {
+ fail = av1_denoiser_realloc_svc_helper(cm, denoiser,
+ lst_fb_idx + 1 + svc_buf_shift);
+ if (fail) return 1;
+ }
+ }
+ return 0;
+}
+
+int av1_denoiser_alloc(AV1_COMMON *cm, struct SVC *svc, AV1_DENOISER *denoiser,
+ int use_svc, int noise_sen, int width, int height,
+ int ssx, int ssy, int use_highbitdepth, int border) {
+ int i, layer, fail, init_num_ref_frames;
+ const int legacy_byte_alignment = 0;
+ int num_layers = 1;
+ int scaled_width = width;
+ int scaled_height = height;
+ if (use_svc) {
+ LAYER_CONTEXT *lc = &svc->layer_context[svc->spatial_layer_id *
+ svc->number_temporal_layers +
+ svc->temporal_layer_id];
+ av1_get_layer_resolution(width, height, lc->scaling_factor_num,
+ lc->scaling_factor_den, &scaled_width,
+ &scaled_height);
+ // For SVC: only denoise at most 2 spatial (highest) layers.
+ if (noise_sen >= 2)
+ // Denoise from one spatial layer below the top.
+ svc->first_layer_denoise = AOMMAX(svc->number_spatial_layers - 2, 0);
+ else
+ // Only denoise the top spatial layer.
+ svc->first_layer_denoise = AOMMAX(svc->number_spatial_layers - 1, 0);
+ num_layers = svc->number_spatial_layers - svc->first_layer_denoise;
+ }
+ assert(denoiser != NULL);
+ denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES;
+ init_num_ref_frames = use_svc ? REF_FRAMES : NONSVC_REF_FRAMES;
+ denoiser->num_layers = num_layers;
+ CHECK_MEM_ERROR(cm, denoiser->running_avg_y,
+ aom_calloc(denoiser->num_ref_frames * num_layers,
+ sizeof(denoiser->running_avg_y[0])));
+ CHECK_MEM_ERROR(
+ cm, denoiser->mc_running_avg_y,
+ aom_calloc(num_layers, sizeof(denoiser->mc_running_avg_y[0])));
+
+ for (layer = 0; layer < num_layers; ++layer) {
+ const int denoise_width = (layer == 0) ? width : scaled_width;
+ const int denoise_height = (layer == 0) ? height : scaled_height;
+ for (i = 0; i < init_num_ref_frames; ++i) {
+ fail = aom_alloc_frame_buffer(
+ &denoiser->running_avg_y[i + denoiser->num_ref_frames * layer],
+ denoise_width, denoise_height, ssx, ssy, use_highbitdepth, border,
+ legacy_byte_alignment);
+ if (fail) {
+ av1_denoiser_free(denoiser);
+ return 1;
+ }
+#ifdef OUTPUT_YUV_DENOISED
+ make_grayscale(&denoiser->running_avg_y[i]);
+#endif
+ }
+
+ fail = aom_alloc_frame_buffer(
+ &denoiser->mc_running_avg_y[layer], denoise_width, denoise_height, ssx,
+ ssy, use_highbitdepth, border, legacy_byte_alignment);
+ if (fail) {
+ av1_denoiser_free(denoiser);
+ return 1;
+ }
+ }
+
+ // denoiser->last_source only used for noise_estimation, so only for top
+ // layer.
+ fail =
+ aom_alloc_frame_buffer(&denoiser->last_source, width, height, ssx, ssy,
+ use_highbitdepth, border, legacy_byte_alignment);
+ if (fail) {
+ av1_denoiser_free(denoiser);
+ return 1;
+ }
+#ifdef OUTPUT_YUV_DENOISED
+ make_grayscale(&denoiser->running_avg_y[i]);
+#endif
+ denoiser->frame_buffer_initialized = 1;
+ denoiser->denoising_level = kDenMedium;
+ denoiser->prev_denoising_level = kDenMedium;
+ denoiser->reset = 0;
+ denoiser->current_denoiser_frame = 0;
+ return 0;
+}
+
+void av1_denoiser_free(AV1_DENOISER *denoiser) {
+ int i;
+ if (denoiser == NULL) {
+ return;
+ }
+ denoiser->frame_buffer_initialized = 0;
+ for (i = 0; i < denoiser->num_ref_frames * denoiser->num_layers; ++i) {
+ aom_free_frame_buffer(&denoiser->running_avg_y[i]);
+ }
+ aom_free(denoiser->running_avg_y);
+ denoiser->running_avg_y = NULL;
+
+ for (i = 0; i < denoiser->num_layers; ++i) {
+ aom_free_frame_buffer(&denoiser->mc_running_avg_y[i]);
+ }
+
+ aom_free(denoiser->mc_running_avg_y);
+ denoiser->mc_running_avg_y = NULL;
+ aom_free_frame_buffer(&denoiser->last_source);
+}
+
+// TODO(kyslov) Enable when SVC temporal denosing is implemented
+#if 0
+static void force_refresh_longterm_ref(AV1_COMP *const cpi) {
+ SVC *const svc = &cpi->svc;
+ // If long term reference is used, force refresh of that slot, so
+ // denoiser buffer for long term reference stays in sync.
+ if (svc->use_gf_temporal_ref_current_layer) {
+ int index = svc->spatial_layer_id;
+ if (svc->number_spatial_layers == 3) index = svc->spatial_layer_id - 1;
+ assert(index >= 0);
+ cpi->alt_fb_idx = svc->buffer_gf_temporal_ref[index].idx;
+ cpi->refresh_alt_ref_frame = 1;
+ }
+}
+#endif
+
+void av1_denoiser_set_noise_level(AV1_COMP *const cpi, int noise_level) {
+ AV1_DENOISER *const denoiser = &cpi->denoiser;
+ denoiser->denoising_level = noise_level;
+ if (denoiser->denoising_level > kDenLowLow &&
+ denoiser->prev_denoising_level == kDenLowLow) {
+ denoiser->reset = 1;
+// TODO(kyslov) Enable when SVC temporal denosing is implemented
+#if 0
+ force_refresh_longterm_ref(cpi);
+#endif
+ } else {
+ denoiser->reset = 0;
+ }
+ denoiser->prev_denoising_level = denoiser->denoising_level;
+}
+
+// Scale/increase the partition threshold
+// for denoiser speed-up.
+int64_t av1_scale_part_thresh(int64_t threshold, AV1_DENOISER_LEVEL noise_level,
+ CONTENT_STATE_SB content_state,
+ int temporal_layer_id) {
+ if ((content_state.source_sad == kLowSad && content_state.low_sumdiff) ||
+ (content_state.source_sad == kHighSad && content_state.low_sumdiff) ||
+ (content_state.lighting_change && !content_state.low_sumdiff) ||
+ (noise_level == kDenHigh) || (temporal_layer_id != 0)) {
+ int64_t scaled_thr =
+ (temporal_layer_id < 2) ? (3 * threshold) >> 1 : (7 * threshold) >> 2;
+ return scaled_thr;
+ } else {
+ return (5 * threshold) >> 2;
+ }
+}
+
+// Scale/increase the ac skip threshold for
+// denoiser speed-up.
+int64_t av1_scale_acskip_thresh(int64_t threshold,
+ AV1_DENOISER_LEVEL noise_level, int abs_sumdiff,
+ int temporal_layer_id) {
+ if (noise_level >= kDenLow && abs_sumdiff < 5)
+ return threshold *=
+ (noise_level == kDenLow) ? 2 : (temporal_layer_id == 2) ? 10 : 6;
+ else
+ return threshold;
+}
+
+void av1_denoiser_reset_on_first_frame(AV1_COMP *const cpi) {
+ if (/*av1_denoise_svc_non_key(cpi) &&*/
+ cpi->denoiser.current_denoiser_frame == 0) {
+ cpi->denoiser.reset = 1;
+// TODO(kyslov) Enable when SVC temporal denosing is implemented
+#if 0
+ force_refresh_longterm_ref(cpi);
+#endif
+ }
+}
+
+void av1_denoiser_update_ref_frame(AV1_COMP *const cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ SVC *const svc = &cpi->svc;
+
+ if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
+ cpi->denoiser.denoising_level > kDenLowLow) {
+ int svc_refresh_denoiser_buffers = 0;
+ int denoise_svc_second_layer = 0;
+ FRAME_TYPE frame_type = cm->current_frame.frame_type == INTRA_ONLY_FRAME
+ ? KEY_FRAME
+ : cm->current_frame.frame_type;
+ cpi->denoiser.current_denoiser_frame++;
+ const int resize_pending =
+ (cpi->resize_pending_params.width &&
+ cpi->resize_pending_params.height &&
+ (cpi->common.width != cpi->resize_pending_params.width ||
+ cpi->common.height != cpi->resize_pending_params.height));
+
+ if (cpi->use_svc) {
+// TODO(kyslov) Enable when SVC temporal denosing is implemented
+#if 0
+ const int svc_buf_shift =
+ svc->number_spatial_layers - svc->spatial_layer_id == 2
+ ? cpi->denoiser.num_ref_frames
+ : 0;
+ int layer =
+ LAYER_IDS_TO_IDX(svc->spatial_layer_id, svc->temporal_layer_id,
+ svc->number_temporal_layers);
+ LAYER_CONTEXT *const lc = &svc->layer_context[layer];
+ svc_refresh_denoiser_buffers =
+ lc->is_key_frame || svc->spatial_layer_sync[svc->spatial_layer_id];
+ denoise_svc_second_layer =
+ svc->number_spatial_layers - svc->spatial_layer_id == 2 ? 1 : 0;
+ // Check if we need to allocate extra buffers in the denoiser
+ // for refreshed frames.
+ if (av1_denoiser_realloc_svc(cm, &cpi->denoiser, svc, svc_buf_shift,
+ cpi->refresh_alt_ref_frame,
+ cpi->refresh_golden_frame,
+ cpi->refresh_last_frame, cpi->alt_fb_idx,
+ cpi->gld_fb_idx, cpi->lst_fb_idx))
+ aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to re-allocate denoiser for SVC");
+#endif
+ }
+ av1_denoiser_update_frame_info(
+ &cpi->denoiser, *cpi->source, svc, frame_type,
+ cpi->refresh_frame.alt_ref_frame, cpi->refresh_frame.golden_frame, 1,
+ svc->ref_idx[6], svc->ref_idx[3], svc->ref_idx[0], resize_pending,
+ svc_refresh_denoiser_buffers, denoise_svc_second_layer);
+ }
+}
+
+#ifdef OUTPUT_YUV_DENOISED
+static void make_grayscale(YV12_BUFFER_CONFIG *yuv) {
+ int r, c;
+ uint8_t *u = yuv->u_buffer;
+ uint8_t *v = yuv->v_buffer;
+
+ for (r = 0; r < yuv->uv_height; ++r) {
+ for (c = 0; c < yuv->uv_width; ++c) {
+ u[c] = UINT8_MAX / 2;
+ v[c] = UINT8_MAX / 2;
+ }
+ u += yuv->uv_stride;
+ v += yuv->uv_stride;
+ }
+}
+
+void aom_write_yuv_frame(FILE *yuv_file, YV12_BUFFER_CONFIG *s) {
+ unsigned char *src = s->y_buffer;
+ int h = s->y_crop_height;
+
+ do {
+ fwrite(src, s->y_width, 1, yuv_file);
+ src += s->y_stride;
+ } while (--h);
+
+ src = s->u_buffer;
+ h = s->uv_crop_height;
+
+ do {
+ fwrite(src, s->uv_width, 1, yuv_file);
+ src += s->uv_stride;
+ } while (--h);
+
+ src = s->v_buffer;
+ h = s->uv_crop_height;
+
+ do {
+ fwrite(src, s->uv_width, 1, yuv_file);
+ src += s->uv_stride;
+ } while (--h);
+}
+#endif
diff --git a/av1/encoder/av1_temporal_denoiser.h b/av1/encoder/av1_temporal_denoiser.h
new file mode 100644
index 0000000..71c8c1c
--- /dev/null
+++ b/av1/encoder/av1_temporal_denoiser.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_AV1_TEMPORAL_DENOISER_H_
+#define AOM_AV1_ENCODER_AV1_TEMPORAL_DENOISER_H_
+
+#include "av1/encoder/block.h"
+#include "aom_scale/yv12config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MOTION_MAGNITUDE_THRESHOLD (8 * 3)
+
+// Denoiser is used in non svc real-time mode which does not use alt-ref, so no
+// need to allocate for it, and hence we need MAX_REF_FRAME - 1
+#define NONSVC_REF_FRAMES REF_FRAMES - 1
+
+// Number of frame buffers when SVC is used. [0] for current denoised buffer and
+// [1..8] for REF_FRAMES
+#define SVC_REF_FRAMES 9
+
+typedef enum av1_denoiser_decision {
+ COPY_BLOCK,
+ FILTER_BLOCK,
+ FILTER_ZEROMV_BLOCK
+} AV1_DENOISER_DECISION;
+
+typedef enum av1_denoiser_level {
+ kDenLowLow,
+ kDenLow,
+ kDenMedium,
+ kDenHigh
+} AV1_DENOISER_LEVEL;
+
+typedef struct av1_denoiser {
+ YV12_BUFFER_CONFIG *running_avg_y;
+ YV12_BUFFER_CONFIG *mc_running_avg_y;
+ YV12_BUFFER_CONFIG last_source;
+ int frame_buffer_initialized;
+ int reset;
+ int num_ref_frames;
+ int num_layers;
+ unsigned int current_denoiser_frame;
+ AV1_DENOISER_LEVEL denoising_level;
+ AV1_DENOISER_LEVEL prev_denoising_level;
+} AV1_DENOISER;
+
+typedef struct {
+ int64_t zero_last_cost_orig;
+ unsigned int *ref_frame_cost;
+ int_mv (*frame_mv)[REF_FRAMES];
+ int reuse_inter_pred;
+ TX_SIZE best_tx_size;
+ PREDICTION_MODE best_mode;
+ MV_REFERENCE_FRAME best_ref_frame;
+ int_interpfilters best_pred_filter;
+ uint8_t best_mode_skip_txfm;
+} AV1_PICKMODE_CTX_DEN;
+
+struct AV1_COMP;
+struct SVC;
+
+void av1_denoiser_update_frame_info(
+ AV1_DENOISER *denoiser, YV12_BUFFER_CONFIG src, struct SVC *svc,
+ FRAME_TYPE frame_type, int refresh_alt_ref_frame, int refresh_golden_frame,
+ int refresh_last_frame, int alt_fb_idx, int gld_fb_idx, int lst_fb_idx,
+ int resized, int svc_refresh_denoiser_buffers, int second_spatial_layer);
+
+void av1_denoiser_denoise(struct AV1_COMP *cpi, MACROBLOCK *mb, int mi_row,
+ int mi_col, BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
+ AV1_DENOISER_DECISION *denoiser_decision,
+ int use_gf_temporal_ref);
+
+void av1_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx);
+
+void av1_denoiser_update_frame_stats(MB_MODE_INFO *mi, int64_t sse,
+ PREDICTION_MODE mode,
+ PICK_MODE_CONTEXT *ctx);
+
+int av1_denoiser_realloc_svc(AV1_COMMON *cm, AV1_DENOISER *denoiser,
+ struct SVC *svc, int svc_buf_shift,
+ int refresh_alt, int refresh_gld, int refresh_lst,
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx);
+
+int av1_denoiser_alloc(AV1_COMMON *cm, struct SVC *svc, AV1_DENOISER *denoiser,
+ int use_svc, int noise_sen, int width, int height,
+ int ssx, int ssy, int use_highbitdepth, int border);
+
+#if CONFIG_AV1_TEMPORAL_DENOISING
+// This function is used by both c and sse2 denoiser implementations.
+// Define it as a static function within the scope where av1_denoiser.h
+// is referenced.
+static INLINE int total_adj_strong_thresh(BLOCK_SIZE bs,
+ int increase_denoising) {
+ return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2);
+}
+#endif
+
+void av1_denoiser_free(AV1_DENOISER *denoiser);
+
+void av1_denoiser_set_noise_level(struct AV1_COMP *const cpi, int noise_level);
+
+void av1_denoiser_reset_on_first_frame(struct AV1_COMP *const cpi);
+
+int64_t av1_scale_part_thresh(int64_t threshold, AV1_DENOISER_LEVEL noise_level,
+ CONTENT_STATE_SB content_state,
+ int temporal_layer_id);
+
+int64_t av1_scale_acskip_thresh(int64_t threshold,
+ AV1_DENOISER_LEVEL noise_level, int abs_sumdiff,
+ int temporal_layer_id);
+
+void av1_denoiser_update_ref_frame(struct AV1_COMP *const cpi);
+
+void aom_write_yuv_frame(FILE *yuv_file, YV12_BUFFER_CONFIG *s);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // AOM_AV1_ENCODER_AV1_TEMPORAL_DENOISER_H_
diff --git a/av1/encoder/context_tree.h b/av1/encoder/context_tree.h
index f243233..25b43df 100644
--- a/av1/encoder/context_tree.h
+++ b/av1/encoder/context_tree.h
@@ -60,6 +60,16 @@
int rd_mode_is_ready; // Flag to indicate whether rd pick mode decision has
// been made.
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ int64_t newmv_sse;
+ int64_t zeromv_sse;
+ int64_t zeromv_lastref_sse;
+ PREDICTION_MODE best_sse_inter_mode;
+ int_mv best_sse_mv;
+ MV_REFERENCE_FRAME best_reference_frame;
+ MV_REFERENCE_FRAME best_zeromv_reference_frame;
+ int sb_skip_denoising;
+#endif
} PICK_MODE_CONTEXT;
typedef struct PC_TREE {
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 5edc072..0f16623 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -87,6 +87,10 @@
#define FILE_NAME_LEN 100
#endif
+#ifdef OUTPUT_YUV_DENOISED
+FILE *yuv_denoised_file = NULL;
+#endif
+
static INLINE void Scale2Ratio(AOM_SCALING mode, int *hr, int *hs) {
switch (mode) {
case NORMAL:
@@ -962,6 +966,9 @@
#ifdef OUTPUT_YUV_REC
yuv_rec_file = fopen("rec.yuv", "wb");
#endif
+#ifdef OUTPUT_YUV_DENOISED
+ yuv_denoised_file = fopen("denoised.yuv", "wb");
+#endif
assert(MAX_LAP_BUFFERS >= MAX_LAG_BUFFERS);
int size = get_stats_buf_size(num_lap_buffers, MAX_LAG_BUFFERS);
@@ -1519,6 +1526,10 @@
#endif
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ av1_denoiser_free(&(cpi->denoiser));
+#endif
+
TplParams *const tpl_data = &cpi->tpl_data;
for (int frame = 0; frame < MAX_LAG_BUFFERS; ++frame) {
aom_free(tpl_data->tpl_stats_pool[frame]);
@@ -1578,6 +1589,10 @@
#ifdef OUTPUT_YUV_REC
fclose(yuv_rec_file);
#endif
+
+#ifdef OUTPUT_YUV_DENOISED
+ fclose(yuv_denoised_file);
+#endif
}
static void generate_psnr_packet(AV1_COMP *cpi) {
@@ -1930,6 +1945,22 @@
}
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+static void setup_denoiser_buffer(AV1_COMP *cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ if (cpi->oxcf.noise_sensitivity > 0 &&
+ !cpi->denoiser.frame_buffer_initialized) {
+ if (av1_denoiser_alloc(
+ cm, &cpi->svc, &cpi->denoiser, cpi->use_svc,
+ cpi->oxcf.noise_sensitivity, cm->width, cm->height,
+ cm->seq_params.subsampling_x, cm->seq_params.subsampling_y,
+ cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS))
+ aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to allocate denoiser");
+ }
+}
+#endif
+
// Returns 1 if the assigned width or height was <= 0.
int av1_set_size_literal(AV1_COMP *cpi, int width, int height) {
AV1_COMMON *cm = &cpi->common;
@@ -1943,6 +1974,10 @@
cm->width = width;
cm->height = height;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ setup_denoiser_buffer(cpi);
+#endif
+
if (initial_dimensions->width && initial_dimensions->height &&
(cm->width > initial_dimensions->width ||
cm->height > initial_dimensions->height)) {
@@ -1975,6 +2010,13 @@
cm->features.coded_lossless && !av1_superres_scaled(cm);
av1_noise_estimate_init(&cpi->noise_estimate, cm->width, cm->height);
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ // Reset the denoiser on the resized frame.
+ if (cpi->oxcf.noise_sensitivity > 0) {
+ av1_denoiser_free(&(cpi->denoiser));
+ setup_denoiser_buffer(cpi);
+ }
+#endif
}
set_mv_search_params(cpi);
@@ -2245,6 +2287,11 @@
av1_update_noise_estimate(cpi);
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && cpi->use_svc)
+ av1_denoiser_reset_on_first_frame(cpi);
+#endif
+
// For 1 spatial layer encoding: if the (non-LAST) reference has different
// resolution from the source then disable that reference. This is to avoid
// significant increase in encode time from scaling the references in
@@ -2639,6 +2686,14 @@
return err;
}
+#ifdef OUTPUT_YUV_DENOISED
+ const AV1EncoderConfig *const oxcf = &cpi->oxcf;
+ if (oxcf->noise_sensitivity > 0 && denoise_svc(cpi)) {
+ aom_write_yuv_frame(yuv_denoised_file,
+ &cpi->denoiser.running_avg_y[INTRA_FRAME]);
+ }
+#endif
+
AV1_COMMON *const cm = &cpi->common;
SequenceHeader *const seq_params = &cm->seq_params;
@@ -2961,6 +3016,10 @@
// for the purpose to verify no mismatch between encoder and decoder.
if (cm->show_frame) cpi->last_show_frame_buf = cm->cur_frame;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ av1_denoiser_update_ref_frame(cpi);
+#endif
+
refresh_reference_frames(cpi);
// Since we allocate a spot for the OVERLAY frame in the gf group, we need
@@ -3147,6 +3206,9 @@
if (frame_is_intra_only(cm) == 0) {
release_scaled_references(cpi);
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ av1_denoiser_update_ref_frame(cpi);
+#endif
// NOTE: Save the new show frame buffer index for --test-code=warn, i.e.,
// for the purpose to verify no mismatch between encoder and decoder.
@@ -3316,6 +3378,11 @@
struct aom_usec_timer timer;
aom_usec_timer_start(&timer);
#endif
+
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ setup_denoiser_buffer(cpi);
+#endif
+
#if CONFIG_DENOISE
if (cpi->oxcf.noise_level > 0)
if (apply_denoise_2d(cpi, sd, cpi->oxcf.noise_block_size,
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index c8e934a..2ce143d 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -60,6 +60,9 @@
#if CONFIG_TUNE_VMAF
#include "av1/encoder/tune_vmaf.h"
#endif
+#if CONFIG_AV1_TEMPORAL_DENOISING
+#include "av1/encoder/av1_temporal_denoiser.h"
+#endif
#include "aom/internal/aom_codec_internal.h"
#include "aom_util/aom_thread.h"
@@ -876,6 +879,10 @@
int noise_block_size;
#endif
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ // Noise sensitivity.
+ int noise_sensitivity;
+#endif
// Bit mask to specify which tier each of the 32 possible operating points
// conforms to.
unsigned int tier_mask;
@@ -2644,6 +2651,13 @@
*/
NOISE_ESTIMATE noise_estimate;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ /*!
+ * Temporal Denoiser
+ */
+ AV1_DENOISER denoiser;
+#endif
+
/*!
* Count on how many consecutive times a block uses small/zeromv for encoding
* in a scale of 8x8 block.
@@ -3214,6 +3228,13 @@
cm->show_frame;
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+static INLINE int denoise_svc(const struct AV1_COMP *const cpi) {
+ return (!cpi->use_svc || (cpi->use_svc && cpi->svc.spatial_layer_id >=
+ cpi->svc.first_layer_denoise));
+}
+#endif
+
#if CONFIG_COLLECT_PARTITION_STATS == 2
static INLINE void av1_print_fr_partition_timing_stats(
const FramePartitionTimingStats *part_stats, const char *filename) {
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index ad0be35..3b42353 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -639,9 +639,20 @@
rd_stats->sse = sse;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
+ cpi->oxcf.speed > 5)
+ ac_thr = av1_scale_acskip_thresh(ac_thr, cpi->denoiser.denoising_level,
+ (abs(sum) >> (bw + bh)),
+ cpi->svc.temporal_layer_id);
+ else
+ ac_thr *= ac_thr_factor(cpi->oxcf.speed, cpi->common.width,
+ cpi->common.height, abs(sum) >> (bw + bh));
+#else
ac_thr *= ac_thr_factor(cpi->oxcf.speed, cpi->common.width,
cpi->common.height, abs(sum) >> (bw + bh));
+#endif
tx_size = calculate_tx_size(cpi, bsize, x, var, sse);
// The code below for setting skip flag assumes tranform size of at least 8x8,
// so force this lower limit on transform.
@@ -1280,6 +1291,92 @@
}
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+static void av1_pickmode_ctx_den_update(
+ AV1_PICKMODE_CTX_DEN *ctx_den, int64_t zero_last_cost_orig,
+ unsigned int ref_frame_cost[REF_FRAMES],
+ int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES], int reuse_inter_pred,
+ BEST_PICKMODE *bp) {
+ ctx_den->zero_last_cost_orig = zero_last_cost_orig;
+ ctx_den->ref_frame_cost = ref_frame_cost;
+ ctx_den->frame_mv = frame_mv;
+ ctx_den->reuse_inter_pred = reuse_inter_pred;
+ ctx_den->best_tx_size = bp->best_tx_size;
+ ctx_den->best_mode = bp->best_mode;
+ ctx_den->best_ref_frame = bp->best_ref_frame;
+ ctx_den->best_pred_filter = bp->best_pred_filter;
+ ctx_den->best_mode_skip_txfm = bp->best_mode_skip_txfm;
+}
+
+static void recheck_zeromv_after_denoising(
+ AV1_COMP *cpi, MB_MODE_INFO *const mi, MACROBLOCK *x, MACROBLOCKD *const xd,
+ AV1_DENOISER_DECISION decision, AV1_PICKMODE_CTX_DEN *ctx_den,
+ struct buf_2d yv12_mb[4][MAX_MB_PLANE], RD_STATS *best_rdc,
+ BEST_PICKMODE *best_pickmode, BLOCK_SIZE bsize, int mi_row, int mi_col) {
+ // If INTRA or GOLDEN reference was selected, re-evaluate ZEROMV on
+ // denoised result. Only do this under noise conditions, and if rdcost of
+ // ZEROMV onoriginal source is not significantly higher than rdcost of best
+ // mode.
+ if (cpi->noise_estimate.enabled && cpi->noise_estimate.level > kLow &&
+ ctx_den->zero_last_cost_orig < (best_rdc->rdcost << 3) &&
+ ((ctx_den->best_ref_frame == INTRA_FRAME && decision >= FILTER_BLOCK) ||
+ (ctx_den->best_ref_frame == GOLDEN_FRAME &&
+ cpi->svc.number_spatial_layers == 1 &&
+ decision == FILTER_ZEROMV_BLOCK))) {
+ // Check if we should pick ZEROMV on denoised signal.
+ AV1_COMMON *const cm = &cpi->common;
+ RD_STATS this_rdc;
+ const ModeCosts *mode_costs = &x->mode_costs;
+ TxfmSearchInfo *txfm_info = &x->txfm_search_info;
+ MB_MODE_INFO_EXT *const mbmi_ext = &x->mbmi_ext;
+
+ mi->mode = GLOBALMV;
+ mi->ref_frame[0] = LAST_FRAME;
+ mi->ref_frame[1] = NONE_FRAME;
+ set_ref_ptrs(cm, xd, mi->ref_frame[0], NONE_FRAME);
+ mi->mv[0].as_int = 0;
+ mi->interp_filters = av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
+ xd->plane[0].pre[0] = yv12_mb[LAST_FRAME][0];
+ av1_enc_build_inter_predictor_y(xd, mi_row, mi_col);
+ model_rd_for_sb_y(cpi, bsize, x, xd, &this_rdc, 1);
+
+ const int16_t mode_ctx =
+ av1_mode_context_analyzer(mbmi_ext->mode_context, mi->ref_frame);
+ this_rdc.rate += cost_mv_ref(mode_costs, GLOBALMV, mode_ctx);
+
+ this_rdc.rate += ctx_den->ref_frame_cost[LAST_FRAME];
+ this_rdc.rdcost = RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist);
+ txfm_info->skip_txfm = this_rdc.skip_txfm;
+ // Don't switch to ZEROMV if the rdcost for ZEROMV on denoised source
+ // is higher than best_ref mode (on original source).
+ if (this_rdc.rdcost > best_rdc->rdcost) {
+ this_rdc = *best_rdc;
+ mi->mode = best_pickmode->best_mode;
+ mi->ref_frame[0] = best_pickmode->best_ref_frame;
+ set_ref_ptrs(cm, xd, mi->ref_frame[0], NONE_FRAME);
+ mi->interp_filters = best_pickmode->best_pred_filter;
+ if (best_pickmode->best_ref_frame == INTRA_FRAME) {
+ mi->mv[0].as_int = INVALID_MV;
+ } else {
+ mi->mv[0].as_int = ctx_den
+ ->frame_mv[best_pickmode->best_mode]
+ [best_pickmode->best_ref_frame]
+ .as_int;
+ if (ctx_den->reuse_inter_pred) {
+ xd->plane[0].pre[0] = yv12_mb[GOLDEN_FRAME][0];
+ av1_enc_build_inter_predictor_y(xd, mi_row, mi_col);
+ }
+ }
+ mi->tx_size = best_pickmode->best_tx_size;
+ txfm_info->skip_txfm = best_pickmode->best_mode_skip_txfm;
+ } else {
+ ctx_den->best_ref_frame = LAST_FRAME;
+ *best_rdc = this_rdc;
+ }
+ }
+}
+#endif // CONFIG_AV1_TEMPORAL_DENOISING
+
static INLINE int get_force_skip_low_temp_var_small_sb(uint8_t *variance_low,
int mi_row, int mi_col,
BLOCK_SIZE bsize) {
@@ -2009,6 +2106,17 @@
const int mi_row = xd->mi_row;
const int mi_col = xd->mi_col;
int use_modeled_non_rd_cost = 0;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ const int denoise_recheck_zeromv = 1;
+ AV1_PICKMODE_CTX_DEN ctx_den;
+ int64_t zero_last_cost_orig = INT64_MAX;
+ int denoise_svc_pickmode = 1;
+ const int resize_pending =
+ (cpi->resize_pending_params.width && cpi->resize_pending_params.height &&
+ (cpi->common.width != cpi->resize_pending_params.width ||
+ cpi->common.height != cpi->resize_pending_params.height));
+
+#endif
init_best_pickmode(&best_pickmode);
@@ -2042,6 +2150,14 @@
mi->ref_frame[0] = NONE_FRAME;
mi->ref_frame[1] = NONE_FRAME;
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0) {
+ // if (cpi->use_svc) denoise_svc_pickmode = av1_denoise_svc_non_key(cpi);
+ if (cpi->denoiser.denoising_level > kDenLowLow && denoise_svc_pickmode)
+ av1_denoiser_reset_frame_stats(ctx);
+ }
+#endif
+
const int gf_temporal_ref = is_same_gf_and_last_scale(cm);
get_ref_frame_use_mask(cpi, x, mi, mi_row, mi_col, bsize, gf_temporal_ref,
@@ -2232,6 +2348,7 @@
const int skip_ctx = av1_get_skip_txfm_context(xd);
const int skip_txfm_cost = mode_costs->skip_txfm_cost[skip_ctx][1];
const int no_skip_txfm_cost = mode_costs->skip_txfm_cost[skip_ctx][0];
+ const int64_t sse_y = this_rdc.sse;
if (this_early_term) {
this_rdc.skip_txfm = 1;
this_rdc.rate = skip_txfm_cost;
@@ -2300,6 +2417,17 @@
frame_mv[this_mode][ref_frame].as_mv.col, cpi->speed,
x->source_variance, x->content_state_sb);
}
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc_pickmode &&
+ cpi->denoiser.denoising_level > kDenLowLow) {
+ av1_denoiser_update_frame_stats(mi, sse_y, this_mode, ctx);
+ // Keep track of zero_last cost.
+ if (ref_frame == LAST_FRAME && frame_mv[this_mode][ref_frame].as_int == 0)
+ zero_last_cost_orig = this_rdc.rdcost;
+ }
+#else
+ (void)sse_y;
+#endif
mode_checked[this_mode][ref_frame] = 1;
#if COLLECT_PICK_MODE_STAT
@@ -2366,6 +2494,25 @@
pd->dst.stride, bw, bh);
}
}
+
+#if CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && resize_pending == 0 &&
+ denoise_svc_pickmode && cpi->denoiser.denoising_level > kDenLowLow &&
+ cpi->denoiser.reset == 0) {
+ AV1_DENOISER_DECISION decision = COPY_BLOCK;
+ ctx->sb_skip_denoising = 0;
+ av1_pickmode_ctx_den_update(&ctx_den, zero_last_cost_orig, ref_costs_single,
+ frame_mv, reuse_inter_pred, &best_pickmode);
+ av1_denoiser_denoise(cpi, x, mi_row, mi_col, bsize, ctx, &decision,
+ gf_temporal_ref);
+ if (denoise_recheck_zeromv)
+ recheck_zeromv_after_denoising(cpi, mi, x, xd, decision, &ctx_den,
+ yv12_mb, &best_rdc, &best_pickmode, bsize,
+ mi_row, mi_col);
+ best_pickmode.best_ref_frame = ctx_den.best_ref_frame;
+ }
+#endif
+
if (cpi->sf.inter_sf.adaptive_rd_thresh) {
THR_MODES best_mode_idx =
mode_idx[best_pickmode.best_ref_frame][mode_offset(mi->mode)];
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index 482141c..409394b 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -1026,7 +1026,9 @@
if (speed >= 8) {
sf->rt_sf.estimate_motion_for_var_based_partition = 1;
sf->rt_sf.short_circuit_low_temp_var = 1;
+#if !CONFIG_AV1_TEMPORAL_DENOISING
sf->rt_sf.reuse_inter_pred_nonrd = 1;
+#endif
sf->rt_sf.use_nonrd_altref_frame = 0;
sf->rt_sf.nonrd_prune_ref_frame_search = 2;
sf->rt_sf.nonrd_check_partition_merge_mode = 0;
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c
index e75ad3c..c38ef60 100644
--- a/av1/encoder/svc_layercontext.c
+++ b/av1/encoder/svc_layercontext.c
@@ -311,21 +311,9 @@
av1_restore_layer_context(cpi);
}
-/*!\brief Get resolution for current layer.
- *
- * \ingroup rate_control
- * \param[in] width_org Original width, unscaled
- * \param[in] height_org Original height, unscaled
- * \param[in] num Numerator for the scale ratio
- * \param[in] den Denominator for the scale ratio
- * \param[in] width_out Output width, scaled for current layer
- * \param[in] height_out Output height, scaled for current layer
- *
- * \return Nothing is returned. Instead the scaled width and height are set.
- */
-static void get_layer_resolution(const int width_org, const int height_org,
- const int num, const int den, int *width_out,
- int *height_out) {
+void av1_get_layer_resolution(const int width_org, const int height_org,
+ const int num, const int den, int *width_out,
+ int *height_out) {
int w, h;
if (width_out == NULL || height_out == NULL || den == 0) return;
w = width_org * num / den;
@@ -343,8 +331,8 @@
int width = 0, height = 0;
lc = &svc->layer_context[svc->spatial_layer_id * svc->number_temporal_layers +
svc->temporal_layer_id];
- get_layer_resolution(cpi->oxcf.frm_dim_cfg.width,
- cpi->oxcf.frm_dim_cfg.height, lc->scaling_factor_num,
- lc->scaling_factor_den, &width, &height);
+ av1_get_layer_resolution(cpi->oxcf.frm_dim_cfg.width,
+ cpi->oxcf.frm_dim_cfg.height, lc->scaling_factor_num,
+ lc->scaling_factor_den, &width, &height);
av1_set_size_literal(cpi, width, height);
}
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h
index 3c7ae67..93b5103 100644
--- a/av1/encoder/svc_layercontext.h
+++ b/av1/encoder/svc_layercontext.h
@@ -115,6 +115,7 @@
int spatial_layer_fb[REF_FRAMES];
int temporal_layer_fb[REF_FRAMES];
int num_encoded_top_layer;
+ int first_layer_denoise;
/*!\endcond */
/*!
@@ -255,6 +256,21 @@
*/
int av1_svc_primary_ref_frame(const struct AV1_COMP *const cpi);
+/*!\brief Get resolution for current layer.
+ *
+ * \ingroup SVC
+ * \param[in] width_org Original width, unscaled
+ * \param[in] height_org Original height, unscaled
+ * \param[in] num Numerator for the scale ratio
+ * \param[in] den Denominator for the scale ratio
+ * \param[in] width_out Output width, scaled for current layer
+ * \param[in] height_out Output height, scaled for current layer
+ *
+ * \return Nothing is returned. Instead the scaled width and height are set.
+ */
+void av1_get_layer_resolution(const int width_org, const int height_org,
+ const int num, const int den, int *width_out,
+ int *height_out);
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index c58f16a..7fd94a2 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -372,11 +372,23 @@
!cpi->sf.rt_sf.force_large_partition_blocks)
threshold_base = (5 * threshold_base) >> 2;
}
-
+ // TODO(kyslov) Enable var based partition adjusment on temporal denoising
+#if 0 // CONFIG_AV1_TEMPORAL_DENOISING
+ if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
+ cpi->oxcf.speed > 5 && cpi->denoiser.denoising_level >= kDenLow)
+ threshold_base =
+ av1_scale_part_thresh(threshold_base, cpi->denoiser.denoising_level,
+ content_state, cpi->svc.temporal_layer_id);
+ else
+ threshold_base =
+ scale_part_thresh_content(threshold_base, cpi->oxcf.speed, cm->width,
+ cm->height, cpi->svc.non_reference_frame);
+#else
// Increase base variance threshold based on content_state/sum_diff level.
threshold_base =
scale_part_thresh_content(threshold_base, cpi->oxcf.speed, cm->width,
cm->height, cpi->svc.non_reference_frame);
+#endif
thresholds[0] = threshold_base >> 1;
thresholds[1] = threshold_base;