AV1 RT: Implement AV1 temporal denoising Based on VP9 implementation. To enable: compile with CONFIG_AV1_TEMPORAL_DENOISING flag. Current limitations: - SVC implementation is not complete - NEON and SSE optimization to follow - No skin detection - No impact of Variance based bartition Change-Id: Id70fe4022689c12938159fd4569d5091115f9052
diff --git a/av1/av1.cmake b/av1/av1.cmake index 718254b..eedc5d8 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -271,6 +271,12 @@ "${AOM_ROOT}/av1/encoder/optical_flow.h") endif() +if(CONFIG_AV1_TEMPORAL_DENOISING) + list(APPEND AOM_AV1_ENCODER_SOURCES + "${AOM_ROOT}/av1/encoder/av1_temporal_denoiser.c" + "${AOM_ROOT}/av1/encoder/av1_temporal_denoiser.h") +endif() + list(APPEND AOM_AV1_COMMON_INTRIN_SSE2 "${AOM_ROOT}/av1/common/cdef_block_sse2.c" "${AOM_ROOT}/av1/common/x86/cfl_sse2.c"
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index 3ea3b8f..17250d2 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -962,6 +962,9 @@ oxcf->noise_block_size = extra_cfg->noise_block_size; #endif +#if CONFIG_AV1_TEMPORAL_DENOISING + oxcf->noise_sensitivity = extra_cfg->noise_sensitivity; +#endif // Set Tile related configuration. tile_cfg->num_tile_groups = extra_cfg->num_tg; // In large-scale tile encoding mode, num_tile_groups is always 1.
diff --git a/av1/encoder/av1_noise_estimate.c b/av1/encoder/av1_noise_estimate.c index 0e6ee15..dbc86c5 100644 --- a/av1/encoder/av1_noise_estimate.c +++ b/av1/encoder/av1_noise_estimate.c
@@ -20,6 +20,9 @@ #include "av1/encoder/context_tree.h" #include "av1/encoder/av1_noise_estimate.h" #include "av1/encoder/encoder.h" +#if CONFIG_AV1_TEMPORAL_DENOISING +#include "av1/encoder/av1_temporal_denoiser.h" +#endif #if CONFIG_AV1_TEMPORAL_DENOISING // For SVC: only do noise estimation on top spatial layer.
diff --git a/av1/encoder/av1_temporal_denoiser.c b/av1/encoder/av1_temporal_denoiser.c new file mode 100644 index 0000000..f8cdc69 --- /dev/null +++ b/av1/encoder/av1_temporal_denoiser.c
@@ -0,0 +1,802 @@ +/* + * Copyright (c) 2020, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <assert.h> +#include <limits.h> +#include <math.h> + +#include "config/aom_dsp_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_scale/yv12config.h" +#include "aom/aom_integer.h" +#include "av1/common/reconinter.h" +#include "av1/encoder/reconinter_enc.h" +#include "av1/encoder/context_tree.h" +#include "av1/encoder/av1_temporal_denoiser.h" +#include "av1/encoder/encoder.h" + +#ifdef OUTPUT_YUV_DENOISED +static void make_grayscale(YV12_BUFFER_CONFIG *yuv); +#endif + +static int absdiff_thresh(BLOCK_SIZE bs, int increase_denoising) { + (void)bs; + return 3 + (increase_denoising ? 1 : 0); +} + +static int delta_thresh(BLOCK_SIZE bs, int increase_denoising) { + (void)bs; + (void)increase_denoising; + return 4; +} + +static int noise_motion_thresh(BLOCK_SIZE bs, int increase_denoising) { + (void)bs; + (void)increase_denoising; + return 625; +} + +static unsigned int sse_thresh(BLOCK_SIZE bs, int increase_denoising) { + return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 80 : 40); +} + +static int sse_diff_thresh(BLOCK_SIZE bs, int increase_denoising, + int motion_magnitude) { + if (motion_magnitude > noise_motion_thresh(bs, increase_denoising)) { + if (increase_denoising) + return (1 << num_pels_log2_lookup[bs]) << 2; + else + return 0; + } else { + return (1 << num_pels_log2_lookup[bs]) << 4; + } +} + +static int total_adj_weak_thresh(BLOCK_SIZE bs, int increase_denoising) { + return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2); +} + +// TODO(kyslov): If increase_denoising is enabled in the future, +// we might need to update the code for calculating 'total_adj' in +// case the C code is not bit-exact with corresponding sse2 code. +int av1_denoiser_filter_c(const uint8_t *sig, int sig_stride, + const uint8_t *mc_avg, int mc_avg_stride, + uint8_t *avg, int avg_stride, int increase_denoising, + BLOCK_SIZE bs, int motion_magnitude) { + int r, c; + const uint8_t *sig_start = sig; + const uint8_t *mc_avg_start = mc_avg; + uint8_t *avg_start = avg; + int diff, adj, absdiff, delta; + int adj_val[] = { 3, 4, 6 }; + int total_adj = 0; + int shift_inc = 1; + + // If motion_magnitude is small, making the denoiser more aggressive by + // increasing the adjustment for each level. Add another increment for + // blocks that are labeled for increase denoising. + if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) { + if (increase_denoising) { + shift_inc = 2; + } + adj_val[0] += shift_inc; + adj_val[1] += shift_inc; + adj_val[2] += shift_inc; + } + + // First attempt to apply a strong temporal denoising filter. + for (r = 0; r < block_size_high[bs]; ++r) { + for (c = 0; c < block_size_wide[bs]; ++c) { + diff = mc_avg[c] - sig[c]; + absdiff = abs(diff); + + if (absdiff <= absdiff_thresh(bs, increase_denoising)) { + avg[c] = mc_avg[c]; + total_adj += diff; + } else { + switch (absdiff) { + case 4: + case 5: + case 6: + case 7: adj = adj_val[0]; break; + case 8: + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: adj = adj_val[1]; break; + default: adj = adj_val[2]; + } + if (diff > 0) { + avg[c] = AOMMIN(UINT8_MAX, sig[c] + adj); + total_adj += adj; + } else { + avg[c] = AOMMAX(0, sig[c] - adj); + total_adj -= adj; + } + } + } + sig += sig_stride; + avg += avg_stride; + mc_avg += mc_avg_stride; + } + + // If the strong filter did not modify the signal too much, we're all set. + if (abs(total_adj) <= total_adj_strong_thresh(bs, increase_denoising)) { + return FILTER_BLOCK; + } + + // Otherwise, we try to dampen the filter if the delta is not too high. + delta = ((abs(total_adj) - total_adj_strong_thresh(bs, increase_denoising)) >> + num_pels_log2_lookup[bs]) + + 1; + + if (delta >= delta_thresh(bs, increase_denoising)) { + return COPY_BLOCK; + } + + mc_avg = mc_avg_start; + avg = avg_start; + sig = sig_start; + for (r = 0; r < block_size_high[bs]; ++r) { + for (c = 0; c < block_size_wide[bs]; ++c) { + diff = mc_avg[c] - sig[c]; + adj = abs(diff); + if (adj > delta) { + adj = delta; + } + if (diff > 0) { + // Diff positive means we made positive adjustment above + // (in first try/attempt), so now make negative adjustment to bring + // denoised signal down. + avg[c] = AOMMAX(0, avg[c] - adj); + total_adj -= adj; + } else { + // Diff negative means we made negative adjustment above + // (in first try/attempt), so now make positive adjustment to bring + // denoised signal up. + avg[c] = AOMMIN(UINT8_MAX, avg[c] + adj); + total_adj += adj; + } + } + sig += sig_stride; + avg += avg_stride; + mc_avg += mc_avg_stride; + } + + // We can use the filter if it has been sufficiently dampened + if (abs(total_adj) <= total_adj_weak_thresh(bs, increase_denoising)) { + return FILTER_BLOCK; + } + return COPY_BLOCK; +} + +static uint8_t *block_start(uint8_t *framebuf, int stride, int mi_row, + int mi_col) { + return framebuf + (stride * mi_row << 2) + (mi_col << 2); +} + +static AV1_DENOISER_DECISION perform_motion_compensation( + AV1_COMMON *const cm, AV1_DENOISER *denoiser, MACROBLOCK *mb, BLOCK_SIZE bs, + int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx, + int motion_magnitude, int *zeromv_filter, int num_spatial_layers, int width, + int lst_fb_idx, int gld_fb_idx, int use_svc, int spatial_layer, + int use_gf_temporal_ref) { + const int sse_diff = (ctx->newmv_sse == UINT_MAX) + ? 0 + : ((int)ctx->zeromv_sse - (int)ctx->newmv_sse); + int frame; + int denoise_layer_idx = 0; + MACROBLOCKD *filter_mbd = &mb->e_mbd; + MB_MODE_INFO *mi = filter_mbd->mi[0]; + MB_MODE_INFO saved_mi; + int i; + struct buf_2d saved_dst[MAX_MB_PLANE]; + struct buf_2d saved_pre[MAX_MB_PLANE]; + // const RefBuffer *saved_block_refs[2]; + MV_REFERENCE_FRAME saved_frame; + + frame = ctx->best_reference_frame; + + saved_mi = *mi; + + // Avoid denoising small blocks. When noise > kDenLow or frame width > 480, + // denoise 16x16 blocks. + if (bs == BLOCK_8X8 || bs == BLOCK_8X16 || bs == BLOCK_16X8 || + (bs == BLOCK_16X16 && width > 480 && + denoiser->denoising_level <= kDenLow)) + return COPY_BLOCK; + + // If the best reference frame uses inter-prediction and there is enough of a + // difference in sum-squared-error, use it. + if (frame != INTRA_FRAME && frame != ALTREF_FRAME && frame != GOLDEN_FRAME && + sse_diff > sse_diff_thresh(bs, increase_denoising, motion_magnitude)) { + mi->ref_frame[0] = ctx->best_reference_frame; + mi->mode = ctx->best_sse_inter_mode; + mi->mv[0] = ctx->best_sse_mv; + } else { + // Otherwise, use the zero reference frame. + frame = ctx->best_zeromv_reference_frame; + ctx->newmv_sse = ctx->zeromv_sse; + // Bias to last reference. + if ((num_spatial_layers > 1 && !use_gf_temporal_ref) || + frame == ALTREF_FRAME || + (frame == GOLDEN_FRAME && use_gf_temporal_ref) || + (frame != LAST_FRAME && + ((ctx->zeromv_lastref_sse<(5 * ctx->zeromv_sse)>> 2) || + denoiser->denoising_level >= kDenHigh))) { + frame = LAST_FRAME; + ctx->newmv_sse = ctx->zeromv_lastref_sse; + } + mi->ref_frame[0] = frame; + mi->mode = GLOBALMV; + mi->mv[0].as_int = 0; + ctx->best_sse_inter_mode = GLOBALMV; + ctx->best_sse_mv.as_int = 0; + *zeromv_filter = 1; + if (denoiser->denoising_level > kDenMedium) { + motion_magnitude = 0; + } + } + + saved_frame = frame; + // When using SVC, we need to map REF_FRAME to the frame buffer index. + if (use_svc) { + if (frame == LAST_FRAME) + frame = lst_fb_idx + 1; + else if (frame == GOLDEN_FRAME) + frame = gld_fb_idx + 1; + // Shift for the second spatial layer. + if (num_spatial_layers - spatial_layer == 2) + frame = frame + denoiser->num_ref_frames; + denoise_layer_idx = num_spatial_layers - spatial_layer - 1; + } + + // Force copy (no denoise, copy source in denoised buffer) if + // running_avg_y[frame] is NULL. + if (denoiser->running_avg_y[frame].buffer_alloc == NULL) { + // Restore everything to its original state + *mi = saved_mi; + return COPY_BLOCK; + } + + if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) { + // Restore everything to its original state + *mi = saved_mi; + return COPY_BLOCK; + } + if (motion_magnitude > (noise_motion_thresh(bs, increase_denoising) << 3)) { + // Restore everything to its original state + *mi = saved_mi; + return COPY_BLOCK; + } + + // We will restore these after motion compensation. + for (i = 0; i < MAX_MB_PLANE; ++i) { + saved_pre[i] = filter_mbd->plane[i].pre[0]; + saved_dst[i] = filter_mbd->plane[i].dst; + } + + // Set the pointers in the MACROBLOCKD to point to the buffers in the denoiser + // struct. + set_ref_ptrs(cm, filter_mbd, saved_frame, NONE); + av1_setup_pre_planes(filter_mbd, 0, &(denoiser->running_avg_y[frame]), mi_row, + mi_col, filter_mbd->block_ref_scale_factors[0], 1); + av1_setup_dst_planes(filter_mbd->plane, bs, + &(denoiser->mc_running_avg_y[denoise_layer_idx]), mi_row, + mi_col, 0, 1); + + av1_enc_build_inter_predictor_y(filter_mbd, mi_row, mi_col); + + // Restore everything to its original state + *mi = saved_mi; + for (i = 0; i < MAX_MB_PLANE; ++i) { + filter_mbd->plane[i].pre[0] = saved_pre[i]; + filter_mbd->plane[i].dst = saved_dst[i]; + } + + return FILTER_BLOCK; +} + +void av1_denoiser_denoise(AV1_COMP *cpi, MACROBLOCK *mb, int mi_row, int mi_col, + BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx, + AV1_DENOISER_DECISION *denoiser_decision, + int use_gf_temporal_ref) { + int mv_col, mv_row; + int motion_magnitude = 0; + int zeromv_filter = 0; + AV1_DENOISER *denoiser = &cpi->denoiser; + AV1_DENOISER_DECISION decision = COPY_BLOCK; + + const int shift = + cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id == 2 + ? denoiser->num_ref_frames + : 0; + YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME + shift]; + const int denoise_layer_index = + cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id - 1; + YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y[denoise_layer_index]; + uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col); + + uint8_t *mc_avg_start = + block_start(mc_avg.y_buffer, mc_avg.y_stride, mi_row, mi_col); + struct buf_2d src = mb->plane[0].src; + int increase_denoising = 0; + int last_is_reference = cpi->ref_frame_flags & AOM_LAST_FLAG; + mv_col = ctx->best_sse_mv.as_mv.col; + mv_row = ctx->best_sse_mv.as_mv.row; + motion_magnitude = mv_row * mv_row + mv_col * mv_col; + + if (denoiser->denoising_level == kDenHigh) increase_denoising = 1; + + // Copy block if LAST_FRAME is not a reference. + // Last doesn't always exist when SVC layers are dynamically changed, e.g. top + // spatial layer doesn't have last reference when it's brought up for the + // first time on the fly. + if (last_is_reference && denoiser->denoising_level >= kDenLow && + !ctx->sb_skip_denoising) + decision = perform_motion_compensation( + &cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx, + motion_magnitude, &zeromv_filter, cpi->svc.number_spatial_layers, + cpi->source->y_width, cpi->svc.ref_idx[0], cpi->svc.ref_idx[3], + cpi->use_svc, cpi->svc.spatial_layer_id, use_gf_temporal_ref); + + if (decision == FILTER_BLOCK) { + decision = av1_denoiser_filter_c(src.buf, src.stride, mc_avg_start, + mc_avg.y_stride, avg_start, avg.y_stride, + increase_denoising, bs, motion_magnitude); + } + + if (decision == FILTER_BLOCK) { + aom_convolve_copy(avg_start, avg.y_stride, src.buf, src.stride, + block_size_wide[bs], block_size_high[bs]); + } else { // COPY_BLOCK + aom_convolve_copy(src.buf, src.stride, avg_start, avg.y_stride, + block_size_wide[bs], block_size_high[bs]); + } + *denoiser_decision = decision; + if (decision == FILTER_BLOCK && zeromv_filter == 1) + *denoiser_decision = FILTER_ZEROMV_BLOCK; +} + +static void copy_frame(YV12_BUFFER_CONFIG *const dest, + const YV12_BUFFER_CONFIG *const src) { + int r; + const uint8_t *srcbuf = src->y_buffer; + uint8_t *destbuf = dest->y_buffer; + + assert(dest->y_width == src->y_width); + assert(dest->y_height == src->y_height); + + for (r = 0; r < dest->y_height; ++r) { + memcpy(destbuf, srcbuf, dest->y_width); + destbuf += dest->y_stride; + srcbuf += src->y_stride; + } +} + +static void swap_frame_buffer(YV12_BUFFER_CONFIG *const dest, + YV12_BUFFER_CONFIG *const src) { + uint8_t *tmp_buf = dest->y_buffer; + assert(dest->y_width == src->y_width); + assert(dest->y_height == src->y_height); + dest->y_buffer = src->y_buffer; + src->y_buffer = tmp_buf; +} + +void av1_denoiser_update_frame_info( + AV1_DENOISER *denoiser, YV12_BUFFER_CONFIG src, struct SVC *svc, + FRAME_TYPE frame_type, int refresh_alt_ref_frame, int refresh_golden_frame, + int refresh_last_frame, int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, + int resized, int svc_refresh_denoiser_buffers, int second_spatial_layer) { + const int shift = second_spatial_layer ? denoiser->num_ref_frames : 0; + // Copy source into denoised reference buffers on KEY_FRAME or + // if the just encoded frame was resized. For SVC, copy source if the base + // spatial layer was key frame. + if (frame_type == KEY_FRAME || resized != 0 || denoiser->reset || + svc_refresh_denoiser_buffers) { + int i; + // Start at 1 so as not to overwrite the INTRA_FRAME + for (i = 1; i < denoiser->num_ref_frames; ++i) { + if (denoiser->running_avg_y[i + shift].buffer_alloc != NULL) + copy_frame(&denoiser->running_avg_y[i + shift], &src); + } + denoiser->reset = 0; + return; + } + + if (svc->external_ref_frame_config) { + int i; + for (i = 0; i < REF_FRAMES; i++) { + if (svc->refresh[svc->spatial_layer_id] & (1 << i)) + copy_frame(&denoiser->running_avg_y[i + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + } else { + // If more than one refresh occurs, must copy frame buffer. + if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) > + 1) { + if (refresh_alt_ref_frame) { + copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + if (refresh_golden_frame) { + copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + if (refresh_last_frame) { + copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + } else { + if (refresh_alt_ref_frame) { + swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + if (refresh_golden_frame) { + swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + if (refresh_last_frame) { + swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1 + shift], + &denoiser->running_avg_y[INTRA_FRAME + shift]); + } + } + } +} + +void av1_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) { + ctx->zeromv_sse = INT64_MAX; + ctx->newmv_sse = INT64_MAX; + ctx->zeromv_lastref_sse = INT64_MAX; + ctx->best_sse_mv.as_int = 0; +} + +void av1_denoiser_update_frame_stats(MB_MODE_INFO *mi, int64_t sse, + PREDICTION_MODE mode, + PICK_MODE_CONTEXT *ctx) { + if (mi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) { + ctx->zeromv_sse = sse; + ctx->best_zeromv_reference_frame = mi->ref_frame[0]; + if (mi->ref_frame[0] == LAST_FRAME) ctx->zeromv_lastref_sse = sse; + } + + if (mi->mv[0].as_int != 0 && sse < ctx->newmv_sse) { + ctx->newmv_sse = sse; + ctx->best_sse_inter_mode = mode; + ctx->best_sse_mv = mi->mv[0]; + ctx->best_reference_frame = mi->ref_frame[0]; + } +} + +static int av1_denoiser_realloc_svc_helper(AV1_COMMON *cm, + AV1_DENOISER *denoiser, int fb_idx) { + int fail = 0; + if (denoiser->running_avg_y[fb_idx].buffer_alloc == NULL) { + fail = aom_alloc_frame_buffer( + &denoiser->running_avg_y[fb_idx], cm->width, cm->height, + cm->seq_params.subsampling_x, cm->seq_params.subsampling_y, + cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS, + cm->features.byte_alignment); + if (fail) { + av1_denoiser_free(denoiser); + return 1; + } + } + return 0; +} + +int av1_denoiser_realloc_svc(AV1_COMMON *cm, AV1_DENOISER *denoiser, + struct SVC *svc, int svc_buf_shift, + int refresh_alt, int refresh_gld, int refresh_lst, + int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) { + int fail = 0; + if (svc->external_ref_frame_config) { + int i; + for (i = 0; i < REF_FRAMES; i++) { + if (cm->current_frame.frame_type == KEY_FRAME || + svc->refresh[svc->spatial_layer_id] & (1 << i)) { + fail = av1_denoiser_realloc_svc_helper(cm, denoiser, + i + 1 + svc_buf_shift); + } + } + } else { + if (refresh_alt) { + // Increase the frame buffer index by 1 to map it to the buffer index in + // the denoiser. + fail = av1_denoiser_realloc_svc_helper(cm, denoiser, + alt_fb_idx + 1 + svc_buf_shift); + if (fail) return 1; + } + if (refresh_gld) { + fail = av1_denoiser_realloc_svc_helper(cm, denoiser, + gld_fb_idx + 1 + svc_buf_shift); + if (fail) return 1; + } + if (refresh_lst) { + fail = av1_denoiser_realloc_svc_helper(cm, denoiser, + lst_fb_idx + 1 + svc_buf_shift); + if (fail) return 1; + } + } + return 0; +} + +int av1_denoiser_alloc(AV1_COMMON *cm, struct SVC *svc, AV1_DENOISER *denoiser, + int use_svc, int noise_sen, int width, int height, + int ssx, int ssy, int use_highbitdepth, int border) { + int i, layer, fail, init_num_ref_frames; + const int legacy_byte_alignment = 0; + int num_layers = 1; + int scaled_width = width; + int scaled_height = height; + if (use_svc) { + LAYER_CONTEXT *lc = &svc->layer_context[svc->spatial_layer_id * + svc->number_temporal_layers + + svc->temporal_layer_id]; + av1_get_layer_resolution(width, height, lc->scaling_factor_num, + lc->scaling_factor_den, &scaled_width, + &scaled_height); + // For SVC: only denoise at most 2 spatial (highest) layers. + if (noise_sen >= 2) + // Denoise from one spatial layer below the top. + svc->first_layer_denoise = AOMMAX(svc->number_spatial_layers - 2, 0); + else + // Only denoise the top spatial layer. + svc->first_layer_denoise = AOMMAX(svc->number_spatial_layers - 1, 0); + num_layers = svc->number_spatial_layers - svc->first_layer_denoise; + } + assert(denoiser != NULL); + denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES; + init_num_ref_frames = use_svc ? REF_FRAMES : NONSVC_REF_FRAMES; + denoiser->num_layers = num_layers; + CHECK_MEM_ERROR(cm, denoiser->running_avg_y, + aom_calloc(denoiser->num_ref_frames * num_layers, + sizeof(denoiser->running_avg_y[0]))); + CHECK_MEM_ERROR( + cm, denoiser->mc_running_avg_y, + aom_calloc(num_layers, sizeof(denoiser->mc_running_avg_y[0]))); + + for (layer = 0; layer < num_layers; ++layer) { + const int denoise_width = (layer == 0) ? width : scaled_width; + const int denoise_height = (layer == 0) ? height : scaled_height; + for (i = 0; i < init_num_ref_frames; ++i) { + fail = aom_alloc_frame_buffer( + &denoiser->running_avg_y[i + denoiser->num_ref_frames * layer], + denoise_width, denoise_height, ssx, ssy, use_highbitdepth, border, + legacy_byte_alignment); + if (fail) { + av1_denoiser_free(denoiser); + return 1; + } +#ifdef OUTPUT_YUV_DENOISED + make_grayscale(&denoiser->running_avg_y[i]); +#endif + } + + fail = aom_alloc_frame_buffer( + &denoiser->mc_running_avg_y[layer], denoise_width, denoise_height, ssx, + ssy, use_highbitdepth, border, legacy_byte_alignment); + if (fail) { + av1_denoiser_free(denoiser); + return 1; + } + } + + // denoiser->last_source only used for noise_estimation, so only for top + // layer. + fail = + aom_alloc_frame_buffer(&denoiser->last_source, width, height, ssx, ssy, + use_highbitdepth, border, legacy_byte_alignment); + if (fail) { + av1_denoiser_free(denoiser); + return 1; + } +#ifdef OUTPUT_YUV_DENOISED + make_grayscale(&denoiser->running_avg_y[i]); +#endif + denoiser->frame_buffer_initialized = 1; + denoiser->denoising_level = kDenMedium; + denoiser->prev_denoising_level = kDenMedium; + denoiser->reset = 0; + denoiser->current_denoiser_frame = 0; + return 0; +} + +void av1_denoiser_free(AV1_DENOISER *denoiser) { + int i; + if (denoiser == NULL) { + return; + } + denoiser->frame_buffer_initialized = 0; + for (i = 0; i < denoiser->num_ref_frames * denoiser->num_layers; ++i) { + aom_free_frame_buffer(&denoiser->running_avg_y[i]); + } + aom_free(denoiser->running_avg_y); + denoiser->running_avg_y = NULL; + + for (i = 0; i < denoiser->num_layers; ++i) { + aom_free_frame_buffer(&denoiser->mc_running_avg_y[i]); + } + + aom_free(denoiser->mc_running_avg_y); + denoiser->mc_running_avg_y = NULL; + aom_free_frame_buffer(&denoiser->last_source); +} + +// TODO(kyslov) Enable when SVC temporal denosing is implemented +#if 0 +static void force_refresh_longterm_ref(AV1_COMP *const cpi) { + SVC *const svc = &cpi->svc; + // If long term reference is used, force refresh of that slot, so + // denoiser buffer for long term reference stays in sync. + if (svc->use_gf_temporal_ref_current_layer) { + int index = svc->spatial_layer_id; + if (svc->number_spatial_layers == 3) index = svc->spatial_layer_id - 1; + assert(index >= 0); + cpi->alt_fb_idx = svc->buffer_gf_temporal_ref[index].idx; + cpi->refresh_alt_ref_frame = 1; + } +} +#endif + +void av1_denoiser_set_noise_level(AV1_COMP *const cpi, int noise_level) { + AV1_DENOISER *const denoiser = &cpi->denoiser; + denoiser->denoising_level = noise_level; + if (denoiser->denoising_level > kDenLowLow && + denoiser->prev_denoising_level == kDenLowLow) { + denoiser->reset = 1; +// TODO(kyslov) Enable when SVC temporal denosing is implemented +#if 0 + force_refresh_longterm_ref(cpi); +#endif + } else { + denoiser->reset = 0; + } + denoiser->prev_denoising_level = denoiser->denoising_level; +} + +// Scale/increase the partition threshold +// for denoiser speed-up. +int64_t av1_scale_part_thresh(int64_t threshold, AV1_DENOISER_LEVEL noise_level, + CONTENT_STATE_SB content_state, + int temporal_layer_id) { + if ((content_state.source_sad == kLowSad && content_state.low_sumdiff) || + (content_state.source_sad == kHighSad && content_state.low_sumdiff) || + (content_state.lighting_change && !content_state.low_sumdiff) || + (noise_level == kDenHigh) || (temporal_layer_id != 0)) { + int64_t scaled_thr = + (temporal_layer_id < 2) ? (3 * threshold) >> 1 : (7 * threshold) >> 2; + return scaled_thr; + } else { + return (5 * threshold) >> 2; + } +} + +// Scale/increase the ac skip threshold for +// denoiser speed-up. +int64_t av1_scale_acskip_thresh(int64_t threshold, + AV1_DENOISER_LEVEL noise_level, int abs_sumdiff, + int temporal_layer_id) { + if (noise_level >= kDenLow && abs_sumdiff < 5) + return threshold *= + (noise_level == kDenLow) ? 2 : (temporal_layer_id == 2) ? 10 : 6; + else + return threshold; +} + +void av1_denoiser_reset_on_first_frame(AV1_COMP *const cpi) { + if (/*av1_denoise_svc_non_key(cpi) &&*/ + cpi->denoiser.current_denoiser_frame == 0) { + cpi->denoiser.reset = 1; +// TODO(kyslov) Enable when SVC temporal denosing is implemented +#if 0 + force_refresh_longterm_ref(cpi); +#endif + } +} + +void av1_denoiser_update_ref_frame(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + SVC *const svc = &cpi->svc; + + if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) && + cpi->denoiser.denoising_level > kDenLowLow) { + int svc_refresh_denoiser_buffers = 0; + int denoise_svc_second_layer = 0; + FRAME_TYPE frame_type = cm->current_frame.frame_type == INTRA_ONLY_FRAME + ? KEY_FRAME + : cm->current_frame.frame_type; + cpi->denoiser.current_denoiser_frame++; + const int resize_pending = + (cpi->resize_pending_params.width && + cpi->resize_pending_params.height && + (cpi->common.width != cpi->resize_pending_params.width || + cpi->common.height != cpi->resize_pending_params.height)); + + if (cpi->use_svc) { +// TODO(kyslov) Enable when SVC temporal denosing is implemented +#if 0 + const int svc_buf_shift = + svc->number_spatial_layers - svc->spatial_layer_id == 2 + ? cpi->denoiser.num_ref_frames + : 0; + int layer = + LAYER_IDS_TO_IDX(svc->spatial_layer_id, svc->temporal_layer_id, + svc->number_temporal_layers); + LAYER_CONTEXT *const lc = &svc->layer_context[layer]; + svc_refresh_denoiser_buffers = + lc->is_key_frame || svc->spatial_layer_sync[svc->spatial_layer_id]; + denoise_svc_second_layer = + svc->number_spatial_layers - svc->spatial_layer_id == 2 ? 1 : 0; + // Check if we need to allocate extra buffers in the denoiser + // for refreshed frames. + if (av1_denoiser_realloc_svc(cm, &cpi->denoiser, svc, svc_buf_shift, + cpi->refresh_alt_ref_frame, + cpi->refresh_golden_frame, + cpi->refresh_last_frame, cpi->alt_fb_idx, + cpi->gld_fb_idx, cpi->lst_fb_idx)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to re-allocate denoiser for SVC"); +#endif + } + av1_denoiser_update_frame_info( + &cpi->denoiser, *cpi->source, svc, frame_type, + cpi->refresh_frame.alt_ref_frame, cpi->refresh_frame.golden_frame, 1, + svc->ref_idx[6], svc->ref_idx[3], svc->ref_idx[0], resize_pending, + svc_refresh_denoiser_buffers, denoise_svc_second_layer); + } +} + +#ifdef OUTPUT_YUV_DENOISED +static void make_grayscale(YV12_BUFFER_CONFIG *yuv) { + int r, c; + uint8_t *u = yuv->u_buffer; + uint8_t *v = yuv->v_buffer; + + for (r = 0; r < yuv->uv_height; ++r) { + for (c = 0; c < yuv->uv_width; ++c) { + u[c] = UINT8_MAX / 2; + v[c] = UINT8_MAX / 2; + } + u += yuv->uv_stride; + v += yuv->uv_stride; + } +} + +void aom_write_yuv_frame(FILE *yuv_file, YV12_BUFFER_CONFIG *s) { + unsigned char *src = s->y_buffer; + int h = s->y_crop_height; + + do { + fwrite(src, s->y_width, 1, yuv_file); + src += s->y_stride; + } while (--h); + + src = s->u_buffer; + h = s->uv_crop_height; + + do { + fwrite(src, s->uv_width, 1, yuv_file); + src += s->uv_stride; + } while (--h); + + src = s->v_buffer; + h = s->uv_crop_height; + + do { + fwrite(src, s->uv_width, 1, yuv_file); + src += s->uv_stride; + } while (--h); +} +#endif
diff --git a/av1/encoder/av1_temporal_denoiser.h b/av1/encoder/av1_temporal_denoiser.h new file mode 100644 index 0000000..71c8c1c --- /dev/null +++ b/av1/encoder/av1_temporal_denoiser.h
@@ -0,0 +1,131 @@ +/* + * Copyright (c) 2020, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AV1_ENCODER_AV1_TEMPORAL_DENOISER_H_ +#define AOM_AV1_ENCODER_AV1_TEMPORAL_DENOISER_H_ + +#include "av1/encoder/block.h" +#include "aom_scale/yv12config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MOTION_MAGNITUDE_THRESHOLD (8 * 3) + +// Denoiser is used in non svc real-time mode which does not use alt-ref, so no +// need to allocate for it, and hence we need MAX_REF_FRAME - 1 +#define NONSVC_REF_FRAMES REF_FRAMES - 1 + +// Number of frame buffers when SVC is used. [0] for current denoised buffer and +// [1..8] for REF_FRAMES +#define SVC_REF_FRAMES 9 + +typedef enum av1_denoiser_decision { + COPY_BLOCK, + FILTER_BLOCK, + FILTER_ZEROMV_BLOCK +} AV1_DENOISER_DECISION; + +typedef enum av1_denoiser_level { + kDenLowLow, + kDenLow, + kDenMedium, + kDenHigh +} AV1_DENOISER_LEVEL; + +typedef struct av1_denoiser { + YV12_BUFFER_CONFIG *running_avg_y; + YV12_BUFFER_CONFIG *mc_running_avg_y; + YV12_BUFFER_CONFIG last_source; + int frame_buffer_initialized; + int reset; + int num_ref_frames; + int num_layers; + unsigned int current_denoiser_frame; + AV1_DENOISER_LEVEL denoising_level; + AV1_DENOISER_LEVEL prev_denoising_level; +} AV1_DENOISER; + +typedef struct { + int64_t zero_last_cost_orig; + unsigned int *ref_frame_cost; + int_mv (*frame_mv)[REF_FRAMES]; + int reuse_inter_pred; + TX_SIZE best_tx_size; + PREDICTION_MODE best_mode; + MV_REFERENCE_FRAME best_ref_frame; + int_interpfilters best_pred_filter; + uint8_t best_mode_skip_txfm; +} AV1_PICKMODE_CTX_DEN; + +struct AV1_COMP; +struct SVC; + +void av1_denoiser_update_frame_info( + AV1_DENOISER *denoiser, YV12_BUFFER_CONFIG src, struct SVC *svc, + FRAME_TYPE frame_type, int refresh_alt_ref_frame, int refresh_golden_frame, + int refresh_last_frame, int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, + int resized, int svc_refresh_denoiser_buffers, int second_spatial_layer); + +void av1_denoiser_denoise(struct AV1_COMP *cpi, MACROBLOCK *mb, int mi_row, + int mi_col, BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx, + AV1_DENOISER_DECISION *denoiser_decision, + int use_gf_temporal_ref); + +void av1_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx); + +void av1_denoiser_update_frame_stats(MB_MODE_INFO *mi, int64_t sse, + PREDICTION_MODE mode, + PICK_MODE_CONTEXT *ctx); + +int av1_denoiser_realloc_svc(AV1_COMMON *cm, AV1_DENOISER *denoiser, + struct SVC *svc, int svc_buf_shift, + int refresh_alt, int refresh_gld, int refresh_lst, + int alt_fb_idx, int gld_fb_idx, int lst_fb_idx); + +int av1_denoiser_alloc(AV1_COMMON *cm, struct SVC *svc, AV1_DENOISER *denoiser, + int use_svc, int noise_sen, int width, int height, + int ssx, int ssy, int use_highbitdepth, int border); + +#if CONFIG_AV1_TEMPORAL_DENOISING +// This function is used by both c and sse2 denoiser implementations. +// Define it as a static function within the scope where av1_denoiser.h +// is referenced. +static INLINE int total_adj_strong_thresh(BLOCK_SIZE bs, + int increase_denoising) { + return (1 << num_pels_log2_lookup[bs]) * (increase_denoising ? 3 : 2); +} +#endif + +void av1_denoiser_free(AV1_DENOISER *denoiser); + +void av1_denoiser_set_noise_level(struct AV1_COMP *const cpi, int noise_level); + +void av1_denoiser_reset_on_first_frame(struct AV1_COMP *const cpi); + +int64_t av1_scale_part_thresh(int64_t threshold, AV1_DENOISER_LEVEL noise_level, + CONTENT_STATE_SB content_state, + int temporal_layer_id); + +int64_t av1_scale_acskip_thresh(int64_t threshold, + AV1_DENOISER_LEVEL noise_level, int abs_sumdiff, + int temporal_layer_id); + +void av1_denoiser_update_ref_frame(struct AV1_COMP *const cpi); + +void aom_write_yuv_frame(FILE *yuv_file, YV12_BUFFER_CONFIG *s); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AV1_ENCODER_AV1_TEMPORAL_DENOISER_H_
diff --git a/av1/encoder/context_tree.h b/av1/encoder/context_tree.h index f243233..25b43df 100644 --- a/av1/encoder/context_tree.h +++ b/av1/encoder/context_tree.h
@@ -60,6 +60,16 @@ int rd_mode_is_ready; // Flag to indicate whether rd pick mode decision has // been made. +#if CONFIG_AV1_TEMPORAL_DENOISING + int64_t newmv_sse; + int64_t zeromv_sse; + int64_t zeromv_lastref_sse; + PREDICTION_MODE best_sse_inter_mode; + int_mv best_sse_mv; + MV_REFERENCE_FRAME best_reference_frame; + MV_REFERENCE_FRAME best_zeromv_reference_frame; + int sb_skip_denoising; +#endif } PICK_MODE_CONTEXT; typedef struct PC_TREE {
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index 5edc072..0f16623 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -87,6 +87,10 @@ #define FILE_NAME_LEN 100 #endif +#ifdef OUTPUT_YUV_DENOISED +FILE *yuv_denoised_file = NULL; +#endif + static INLINE void Scale2Ratio(AOM_SCALING mode, int *hr, int *hs) { switch (mode) { case NORMAL: @@ -962,6 +966,9 @@ #ifdef OUTPUT_YUV_REC yuv_rec_file = fopen("rec.yuv", "wb"); #endif +#ifdef OUTPUT_YUV_DENOISED + yuv_denoised_file = fopen("denoised.yuv", "wb"); +#endif assert(MAX_LAP_BUFFERS >= MAX_LAG_BUFFERS); int size = get_stats_buf_size(num_lap_buffers, MAX_LAG_BUFFERS); @@ -1519,6 +1526,10 @@ #endif } +#if CONFIG_AV1_TEMPORAL_DENOISING + av1_denoiser_free(&(cpi->denoiser)); +#endif + TplParams *const tpl_data = &cpi->tpl_data; for (int frame = 0; frame < MAX_LAG_BUFFERS; ++frame) { aom_free(tpl_data->tpl_stats_pool[frame]); @@ -1578,6 +1589,10 @@ #ifdef OUTPUT_YUV_REC fclose(yuv_rec_file); #endif + +#ifdef OUTPUT_YUV_DENOISED + fclose(yuv_denoised_file); +#endif } static void generate_psnr_packet(AV1_COMP *cpi) { @@ -1930,6 +1945,22 @@ } } +#if CONFIG_AV1_TEMPORAL_DENOISING +static void setup_denoiser_buffer(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + if (cpi->oxcf.noise_sensitivity > 0 && + !cpi->denoiser.frame_buffer_initialized) { + if (av1_denoiser_alloc( + cm, &cpi->svc, &cpi->denoiser, cpi->use_svc, + cpi->oxcf.noise_sensitivity, cm->width, cm->height, + cm->seq_params.subsampling_x, cm->seq_params.subsampling_y, + cm->seq_params.use_highbitdepth, AOM_BORDER_IN_PIXELS)) + aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR, + "Failed to allocate denoiser"); + } +} +#endif + // Returns 1 if the assigned width or height was <= 0. int av1_set_size_literal(AV1_COMP *cpi, int width, int height) { AV1_COMMON *cm = &cpi->common; @@ -1943,6 +1974,10 @@ cm->width = width; cm->height = height; +#if CONFIG_AV1_TEMPORAL_DENOISING + setup_denoiser_buffer(cpi); +#endif + if (initial_dimensions->width && initial_dimensions->height && (cm->width > initial_dimensions->width || cm->height > initial_dimensions->height)) { @@ -1975,6 +2010,13 @@ cm->features.coded_lossless && !av1_superres_scaled(cm); av1_noise_estimate_init(&cpi->noise_estimate, cm->width, cm->height); +#if CONFIG_AV1_TEMPORAL_DENOISING + // Reset the denoiser on the resized frame. + if (cpi->oxcf.noise_sensitivity > 0) { + av1_denoiser_free(&(cpi->denoiser)); + setup_denoiser_buffer(cpi); + } +#endif } set_mv_search_params(cpi); @@ -2245,6 +2287,11 @@ av1_update_noise_estimate(cpi); } +#if CONFIG_AV1_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity > 0 && cpi->use_svc) + av1_denoiser_reset_on_first_frame(cpi); +#endif + // For 1 spatial layer encoding: if the (non-LAST) reference has different // resolution from the source then disable that reference. This is to avoid // significant increase in encode time from scaling the references in @@ -2639,6 +2686,14 @@ return err; } +#ifdef OUTPUT_YUV_DENOISED + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + if (oxcf->noise_sensitivity > 0 && denoise_svc(cpi)) { + aom_write_yuv_frame(yuv_denoised_file, + &cpi->denoiser.running_avg_y[INTRA_FRAME]); + } +#endif + AV1_COMMON *const cm = &cpi->common; SequenceHeader *const seq_params = &cm->seq_params; @@ -2961,6 +3016,10 @@ // for the purpose to verify no mismatch between encoder and decoder. if (cm->show_frame) cpi->last_show_frame_buf = cm->cur_frame; +#if CONFIG_AV1_TEMPORAL_DENOISING + av1_denoiser_update_ref_frame(cpi); +#endif + refresh_reference_frames(cpi); // Since we allocate a spot for the OVERLAY frame in the gf group, we need @@ -3147,6 +3206,9 @@ if (frame_is_intra_only(cm) == 0) { release_scaled_references(cpi); } +#if CONFIG_AV1_TEMPORAL_DENOISING + av1_denoiser_update_ref_frame(cpi); +#endif // NOTE: Save the new show frame buffer index for --test-code=warn, i.e., // for the purpose to verify no mismatch between encoder and decoder. @@ -3316,6 +3378,11 @@ struct aom_usec_timer timer; aom_usec_timer_start(&timer); #endif + +#if CONFIG_AV1_TEMPORAL_DENOISING + setup_denoiser_buffer(cpi); +#endif + #if CONFIG_DENOISE if (cpi->oxcf.noise_level > 0) if (apply_denoise_2d(cpi, sd, cpi->oxcf.noise_block_size,
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index c8e934a..2ce143d 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -60,6 +60,9 @@ #if CONFIG_TUNE_VMAF #include "av1/encoder/tune_vmaf.h" #endif +#if CONFIG_AV1_TEMPORAL_DENOISING +#include "av1/encoder/av1_temporal_denoiser.h" +#endif #include "aom/internal/aom_codec_internal.h" #include "aom_util/aom_thread.h" @@ -876,6 +879,10 @@ int noise_block_size; #endif +#if CONFIG_AV1_TEMPORAL_DENOISING + // Noise sensitivity. + int noise_sensitivity; +#endif // Bit mask to specify which tier each of the 32 possible operating points // conforms to. unsigned int tier_mask; @@ -2644,6 +2651,13 @@ */ NOISE_ESTIMATE noise_estimate; +#if CONFIG_AV1_TEMPORAL_DENOISING + /*! + * Temporal Denoiser + */ + AV1_DENOISER denoiser; +#endif + /*! * Count on how many consecutive times a block uses small/zeromv for encoding * in a scale of 8x8 block. @@ -3214,6 +3228,13 @@ cm->show_frame; } +#if CONFIG_AV1_TEMPORAL_DENOISING +static INLINE int denoise_svc(const struct AV1_COMP *const cpi) { + return (!cpi->use_svc || (cpi->use_svc && cpi->svc.spatial_layer_id >= + cpi->svc.first_layer_denoise)); +} +#endif + #if CONFIG_COLLECT_PARTITION_STATS == 2 static INLINE void av1_print_fr_partition_timing_stats( const FramePartitionTimingStats *part_stats, const char *filename) {
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c index ad0be35..3b42353 100644 --- a/av1/encoder/nonrd_pickmode.c +++ b/av1/encoder/nonrd_pickmode.c
@@ -639,9 +639,20 @@ rd_stats->sse = sse; +#if CONFIG_AV1_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) && + cpi->oxcf.speed > 5) + ac_thr = av1_scale_acskip_thresh(ac_thr, cpi->denoiser.denoising_level, + (abs(sum) >> (bw + bh)), + cpi->svc.temporal_layer_id); + else + ac_thr *= ac_thr_factor(cpi->oxcf.speed, cpi->common.width, + cpi->common.height, abs(sum) >> (bw + bh)); +#else ac_thr *= ac_thr_factor(cpi->oxcf.speed, cpi->common.width, cpi->common.height, abs(sum) >> (bw + bh)); +#endif tx_size = calculate_tx_size(cpi, bsize, x, var, sse); // The code below for setting skip flag assumes tranform size of at least 8x8, // so force this lower limit on transform. @@ -1280,6 +1291,92 @@ } } +#if CONFIG_AV1_TEMPORAL_DENOISING +static void av1_pickmode_ctx_den_update( + AV1_PICKMODE_CTX_DEN *ctx_den, int64_t zero_last_cost_orig, + unsigned int ref_frame_cost[REF_FRAMES], + int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES], int reuse_inter_pred, + BEST_PICKMODE *bp) { + ctx_den->zero_last_cost_orig = zero_last_cost_orig; + ctx_den->ref_frame_cost = ref_frame_cost; + ctx_den->frame_mv = frame_mv; + ctx_den->reuse_inter_pred = reuse_inter_pred; + ctx_den->best_tx_size = bp->best_tx_size; + ctx_den->best_mode = bp->best_mode; + ctx_den->best_ref_frame = bp->best_ref_frame; + ctx_den->best_pred_filter = bp->best_pred_filter; + ctx_den->best_mode_skip_txfm = bp->best_mode_skip_txfm; +} + +static void recheck_zeromv_after_denoising( + AV1_COMP *cpi, MB_MODE_INFO *const mi, MACROBLOCK *x, MACROBLOCKD *const xd, + AV1_DENOISER_DECISION decision, AV1_PICKMODE_CTX_DEN *ctx_den, + struct buf_2d yv12_mb[4][MAX_MB_PLANE], RD_STATS *best_rdc, + BEST_PICKMODE *best_pickmode, BLOCK_SIZE bsize, int mi_row, int mi_col) { + // If INTRA or GOLDEN reference was selected, re-evaluate ZEROMV on + // denoised result. Only do this under noise conditions, and if rdcost of + // ZEROMV onoriginal source is not significantly higher than rdcost of best + // mode. + if (cpi->noise_estimate.enabled && cpi->noise_estimate.level > kLow && + ctx_den->zero_last_cost_orig < (best_rdc->rdcost << 3) && + ((ctx_den->best_ref_frame == INTRA_FRAME && decision >= FILTER_BLOCK) || + (ctx_den->best_ref_frame == GOLDEN_FRAME && + cpi->svc.number_spatial_layers == 1 && + decision == FILTER_ZEROMV_BLOCK))) { + // Check if we should pick ZEROMV on denoised signal. + AV1_COMMON *const cm = &cpi->common; + RD_STATS this_rdc; + const ModeCosts *mode_costs = &x->mode_costs; + TxfmSearchInfo *txfm_info = &x->txfm_search_info; + MB_MODE_INFO_EXT *const mbmi_ext = &x->mbmi_ext; + + mi->mode = GLOBALMV; + mi->ref_frame[0] = LAST_FRAME; + mi->ref_frame[1] = NONE_FRAME; + set_ref_ptrs(cm, xd, mi->ref_frame[0], NONE_FRAME); + mi->mv[0].as_int = 0; + mi->interp_filters = av1_broadcast_interp_filter(EIGHTTAP_REGULAR); + xd->plane[0].pre[0] = yv12_mb[LAST_FRAME][0]; + av1_enc_build_inter_predictor_y(xd, mi_row, mi_col); + model_rd_for_sb_y(cpi, bsize, x, xd, &this_rdc, 1); + + const int16_t mode_ctx = + av1_mode_context_analyzer(mbmi_ext->mode_context, mi->ref_frame); + this_rdc.rate += cost_mv_ref(mode_costs, GLOBALMV, mode_ctx); + + this_rdc.rate += ctx_den->ref_frame_cost[LAST_FRAME]; + this_rdc.rdcost = RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist); + txfm_info->skip_txfm = this_rdc.skip_txfm; + // Don't switch to ZEROMV if the rdcost for ZEROMV on denoised source + // is higher than best_ref mode (on original source). + if (this_rdc.rdcost > best_rdc->rdcost) { + this_rdc = *best_rdc; + mi->mode = best_pickmode->best_mode; + mi->ref_frame[0] = best_pickmode->best_ref_frame; + set_ref_ptrs(cm, xd, mi->ref_frame[0], NONE_FRAME); + mi->interp_filters = best_pickmode->best_pred_filter; + if (best_pickmode->best_ref_frame == INTRA_FRAME) { + mi->mv[0].as_int = INVALID_MV; + } else { + mi->mv[0].as_int = ctx_den + ->frame_mv[best_pickmode->best_mode] + [best_pickmode->best_ref_frame] + .as_int; + if (ctx_den->reuse_inter_pred) { + xd->plane[0].pre[0] = yv12_mb[GOLDEN_FRAME][0]; + av1_enc_build_inter_predictor_y(xd, mi_row, mi_col); + } + } + mi->tx_size = best_pickmode->best_tx_size; + txfm_info->skip_txfm = best_pickmode->best_mode_skip_txfm; + } else { + ctx_den->best_ref_frame = LAST_FRAME; + *best_rdc = this_rdc; + } + } +} +#endif // CONFIG_AV1_TEMPORAL_DENOISING + static INLINE int get_force_skip_low_temp_var_small_sb(uint8_t *variance_low, int mi_row, int mi_col, BLOCK_SIZE bsize) { @@ -2009,6 +2106,17 @@ const int mi_row = xd->mi_row; const int mi_col = xd->mi_col; int use_modeled_non_rd_cost = 0; +#if CONFIG_AV1_TEMPORAL_DENOISING + const int denoise_recheck_zeromv = 1; + AV1_PICKMODE_CTX_DEN ctx_den; + int64_t zero_last_cost_orig = INT64_MAX; + int denoise_svc_pickmode = 1; + const int resize_pending = + (cpi->resize_pending_params.width && cpi->resize_pending_params.height && + (cpi->common.width != cpi->resize_pending_params.width || + cpi->common.height != cpi->resize_pending_params.height)); + +#endif init_best_pickmode(&best_pickmode); @@ -2042,6 +2150,14 @@ mi->ref_frame[0] = NONE_FRAME; mi->ref_frame[1] = NONE_FRAME; +#if CONFIG_AV1_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity > 0) { + // if (cpi->use_svc) denoise_svc_pickmode = av1_denoise_svc_non_key(cpi); + if (cpi->denoiser.denoising_level > kDenLowLow && denoise_svc_pickmode) + av1_denoiser_reset_frame_stats(ctx); + } +#endif + const int gf_temporal_ref = is_same_gf_and_last_scale(cm); get_ref_frame_use_mask(cpi, x, mi, mi_row, mi_col, bsize, gf_temporal_ref, @@ -2232,6 +2348,7 @@ const int skip_ctx = av1_get_skip_txfm_context(xd); const int skip_txfm_cost = mode_costs->skip_txfm_cost[skip_ctx][1]; const int no_skip_txfm_cost = mode_costs->skip_txfm_cost[skip_ctx][0]; + const int64_t sse_y = this_rdc.sse; if (this_early_term) { this_rdc.skip_txfm = 1; this_rdc.rate = skip_txfm_cost; @@ -2300,6 +2417,17 @@ frame_mv[this_mode][ref_frame].as_mv.col, cpi->speed, x->source_variance, x->content_state_sb); } +#if CONFIG_AV1_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc_pickmode && + cpi->denoiser.denoising_level > kDenLowLow) { + av1_denoiser_update_frame_stats(mi, sse_y, this_mode, ctx); + // Keep track of zero_last cost. + if (ref_frame == LAST_FRAME && frame_mv[this_mode][ref_frame].as_int == 0) + zero_last_cost_orig = this_rdc.rdcost; + } +#else + (void)sse_y; +#endif mode_checked[this_mode][ref_frame] = 1; #if COLLECT_PICK_MODE_STAT @@ -2366,6 +2494,25 @@ pd->dst.stride, bw, bh); } } + +#if CONFIG_AV1_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity > 0 && resize_pending == 0 && + denoise_svc_pickmode && cpi->denoiser.denoising_level > kDenLowLow && + cpi->denoiser.reset == 0) { + AV1_DENOISER_DECISION decision = COPY_BLOCK; + ctx->sb_skip_denoising = 0; + av1_pickmode_ctx_den_update(&ctx_den, zero_last_cost_orig, ref_costs_single, + frame_mv, reuse_inter_pred, &best_pickmode); + av1_denoiser_denoise(cpi, x, mi_row, mi_col, bsize, ctx, &decision, + gf_temporal_ref); + if (denoise_recheck_zeromv) + recheck_zeromv_after_denoising(cpi, mi, x, xd, decision, &ctx_den, + yv12_mb, &best_rdc, &best_pickmode, bsize, + mi_row, mi_col); + best_pickmode.best_ref_frame = ctx_den.best_ref_frame; + } +#endif + if (cpi->sf.inter_sf.adaptive_rd_thresh) { THR_MODES best_mode_idx = mode_idx[best_pickmode.best_ref_frame][mode_offset(mi->mode)];
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c index 482141c..409394b 100644 --- a/av1/encoder/speed_features.c +++ b/av1/encoder/speed_features.c
@@ -1026,7 +1026,9 @@ if (speed >= 8) { sf->rt_sf.estimate_motion_for_var_based_partition = 1; sf->rt_sf.short_circuit_low_temp_var = 1; +#if !CONFIG_AV1_TEMPORAL_DENOISING sf->rt_sf.reuse_inter_pred_nonrd = 1; +#endif sf->rt_sf.use_nonrd_altref_frame = 0; sf->rt_sf.nonrd_prune_ref_frame_search = 2; sf->rt_sf.nonrd_check_partition_merge_mode = 0;
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c index e75ad3c..c38ef60 100644 --- a/av1/encoder/svc_layercontext.c +++ b/av1/encoder/svc_layercontext.c
@@ -311,21 +311,9 @@ av1_restore_layer_context(cpi); } -/*!\brief Get resolution for current layer. - * - * \ingroup rate_control - * \param[in] width_org Original width, unscaled - * \param[in] height_org Original height, unscaled - * \param[in] num Numerator for the scale ratio - * \param[in] den Denominator for the scale ratio - * \param[in] width_out Output width, scaled for current layer - * \param[in] height_out Output height, scaled for current layer - * - * \return Nothing is returned. Instead the scaled width and height are set. - */ -static void get_layer_resolution(const int width_org, const int height_org, - const int num, const int den, int *width_out, - int *height_out) { +void av1_get_layer_resolution(const int width_org, const int height_org, + const int num, const int den, int *width_out, + int *height_out) { int w, h; if (width_out == NULL || height_out == NULL || den == 0) return; w = width_org * num / den; @@ -343,8 +331,8 @@ int width = 0, height = 0; lc = &svc->layer_context[svc->spatial_layer_id * svc->number_temporal_layers + svc->temporal_layer_id]; - get_layer_resolution(cpi->oxcf.frm_dim_cfg.width, - cpi->oxcf.frm_dim_cfg.height, lc->scaling_factor_num, - lc->scaling_factor_den, &width, &height); + av1_get_layer_resolution(cpi->oxcf.frm_dim_cfg.width, + cpi->oxcf.frm_dim_cfg.height, lc->scaling_factor_num, + lc->scaling_factor_den, &width, &height); av1_set_size_literal(cpi, width, height); }
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h index 3c7ae67..93b5103 100644 --- a/av1/encoder/svc_layercontext.h +++ b/av1/encoder/svc_layercontext.h
@@ -115,6 +115,7 @@ int spatial_layer_fb[REF_FRAMES]; int temporal_layer_fb[REF_FRAMES]; int num_encoded_top_layer; + int first_layer_denoise; /*!\endcond */ /*! @@ -255,6 +256,21 @@ */ int av1_svc_primary_ref_frame(const struct AV1_COMP *const cpi); +/*!\brief Get resolution for current layer. + * + * \ingroup SVC + * \param[in] width_org Original width, unscaled + * \param[in] height_org Original height, unscaled + * \param[in] num Numerator for the scale ratio + * \param[in] den Denominator for the scale ratio + * \param[in] width_out Output width, scaled for current layer + * \param[in] height_out Output height, scaled for current layer + * + * \return Nothing is returned. Instead the scaled width and height are set. + */ +void av1_get_layer_resolution(const int width_org, const int height_org, + const int num, const int den, int *width_out, + int *height_out); #ifdef __cplusplus } // extern "C" #endif
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c index c58f16a..7fd94a2 100644 --- a/av1/encoder/var_based_part.c +++ b/av1/encoder/var_based_part.c
@@ -372,11 +372,23 @@ !cpi->sf.rt_sf.force_large_partition_blocks) threshold_base = (5 * threshold_base) >> 2; } - + // TODO(kyslov) Enable var based partition adjusment on temporal denoising +#if 0 // CONFIG_AV1_TEMPORAL_DENOISING + if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) && + cpi->oxcf.speed > 5 && cpi->denoiser.denoising_level >= kDenLow) + threshold_base = + av1_scale_part_thresh(threshold_base, cpi->denoiser.denoising_level, + content_state, cpi->svc.temporal_layer_id); + else + threshold_base = + scale_part_thresh_content(threshold_base, cpi->oxcf.speed, cm->width, + cm->height, cpi->svc.non_reference_frame); +#else // Increase base variance threshold based on content_state/sum_diff level. threshold_base = scale_part_thresh_content(threshold_base, cpi->oxcf.speed, cm->width, cm->height, cpi->svc.non_reference_frame); +#endif thresholds[0] = threshold_base >> 1; thresholds[1] = threshold_base;