rtc-svc: Fixes for screen with spatial layers Several fixes to make screen work with spatial layers: -last_source is set based on superframe count to ensure last_source is the previous source for all spatial layers -source_last_TL0 is used to ensure last_source corresponds to the LAST reference. -scene detection is only done on SLO with full resolution, and the state of the detection is passed on to the other spatial layers. -source_sad is allowed for every spatial layer. -cyclic refresh counter on distance to scene change/max_q is added to the layer context and updated per spatial layer. -allow for prediction from lower spatial layer in nonrd_pickmode. Unittest added for screen with spatial layers. No change in stats for default 1 layer encoding. Bug: aomedia:3346 Change-Id: I87961ead6154fc3642071ab67e99a4595565d00d
diff --git a/av1/encoder/aq_cyclicrefresh.c b/av1/encoder/aq_cyclicrefresh.c index 72304b3..e25bbaf 100644 --- a/av1/encoder/aq_cyclicrefresh.c +++ b/av1/encoder/aq_cyclicrefresh.c
@@ -328,7 +328,8 @@ ymis = AOMMIN(mi_params->mi_rows - mi_row, cm->seq_params->mib_size); if (cr->use_block_sad_scene_det && cpi->rc.frames_since_key > 30 && cr->counter_encode_maxq_scene_change > 30 && - cpi->src_sad_blk_64x64 != NULL) { + cpi->src_sad_blk_64x64 != NULL && + cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1) { sb_sad = cpi->src_sad_blk_64x64[sb_col_index + sb_cols * sb_row_index]; int scale = (cm->width * cm->height < 640 * 360) ? 6 : 8; int scale_low = 2; @@ -373,8 +374,7 @@ } static int is_scene_change_detected(AV1_COMP *const cpi) { - return cpi->rc.high_source_sad || - (cpi->ppi->use_svc && cpi->svc.high_source_sad_superframe); + return cpi->rc.high_source_sad; } // Set cyclic refresh parameters.
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index 357f283..44a9356 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -1387,19 +1387,10 @@ // Source may be changed if temporal filtered later. frame_input.source = &source->img; - frame_input.last_source = last_source != NULL ? &last_source->img : NULL; - // For temporal layers: if LAST reference is not the previous frame set the - // last_source to the source frame corresponding to the LAST reference. - // TODO(marpan): For now only do this for single spatial layer: the case of - // number_spatial_layers > 1 will be handled in another change. - if (cm->current_frame.frame_number > 0 && - cpi->svc.number_temporal_layers > 1 && - cpi->svc.number_spatial_layers == 1) { - const int buffslot_last = cpi->ppi->rtc_ref.ref_idx[0]; // index 0 is LAST - if (cpi->svc.frame_number_buffslot[buffslot_last] < - cm->current_frame.frame_number - 1) - frame_input.last_source = &cpi->svc.source_last_ref; - } + if (cpi->ppi->use_svc && last_source != NULL) + av1_svc_set_last_source(cpi, &frame_input, &last_source->img); + else + frame_input.last_source = last_source != NULL ? &last_source->img : NULL; frame_input.ts_duration = source->ts_end - source->ts_start; // Save unfiltered source. It is used in av1_get_second_pass_params(). cpi->unfiltered_source = frame_input.source; @@ -1689,16 +1680,16 @@ is_frame_droppable(&cpi->ppi->rtc_ref, &ext_flags->refresh_frame); } - // For temporal layers: keep track of the source corresponding to the - // refresh of LAST reference (index 0). Note if temporal filter or denoising - // is on, the source will be modified during encodiing, but for now keep - // this as is. - if (cpi->svc.number_temporal_layers > 1 && - cpi->svc.number_spatial_layers == 1 && - cpi->ppi->rtc_ref.refresh[cpi->ppi->rtc_ref.ref_idx[0]]) { - aom_yv12_copy_y(cpi->source, &cpi->svc.source_last_ref); - aom_yv12_copy_u(cpi->source, &cpi->svc.source_last_ref); - aom_yv12_copy_v(cpi->source, &cpi->svc.source_last_ref); + // For SVC: keep track of the (unscaled) source corresponding to the + // refresh of LAST reference (base temporal layer- TL0). Copy only for the + // top spatial enhancement layer only so all spatial layers of the next + // superframe have last_source to be aligned with previous TL0 superframe. + if (cpi->ppi->use_svc && + cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 && + cpi->svc.temporal_layer_id == 0) { + aom_yv12_copy_y(cpi->unscaled_source, &cpi->svc.source_last_TL0); + aom_yv12_copy_u(cpi->unscaled_source, &cpi->svc.source_last_TL0); + aom_yv12_copy_v(cpi->unscaled_source, &cpi->svc.source_last_TL0); } return AOM_CODEC_OK;
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 3db5689..59ecc75 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -971,6 +971,8 @@ static AOM_INLINE bool is_calc_src_content_needed(AV1_COMP *cpi, MACROBLOCK *const x, int mi_row, int mi_col) { + if (cpi->svc.spatial_layer_id < cpi->svc.number_spatial_layers - 1) + return true; const uint64_t curr_sb_sad = get_sb_source_sad(cpi, mi_row, mi_col); if (curr_sb_sad == UINT64_MAX) return true; if (curr_sb_sad == 0) { @@ -1017,15 +1019,16 @@ TileDataEnc *tile_data, int mi_row, int mi_col) { AV1_COMMON *const cm = &cpi->common; - if (cm->current_frame.frame_type == KEY_FRAME) { + if (cm->current_frame.frame_type == KEY_FRAME || + (cpi->ppi->use_svc && + cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame)) { assert(x->content_state_sb.source_sad_nonrd == kMedSad); assert(x->content_state_sb.source_sad_rd == kMedSad); return; } bool calc_src_content = false; - if (cpi->sf.rt_sf.source_metrics_sb_nonrd && - cpi->svc.number_spatial_layers <= 1) { + if (cpi->sf.rt_sf.source_metrics_sb_nonrd) { if (!cpi->sf.rt_sf.check_scene_detection || cpi->rc.frame_source_sad > 0) { calc_src_content = is_calc_src_content_needed(cpi, x, mi_row, mi_col); } else {
diff --git a/av1/encoder/encodeframe_utils.c b/av1/encoder/encodeframe_utils.c index 624baa6..bae449c 100644 --- a/av1/encoder/encodeframe_utils.c +++ b/av1/encoder/encodeframe_utils.c
@@ -1426,7 +1426,7 @@ x->content_state_sb.low_sumdiff = 1; if (!cpi->sf.rt_sf.use_rtc_tf || cpi->rc.high_source_sad || - cpi->rc.frame_source_sad > 20000) + cpi->rc.frame_source_sad > 20000 || cpi->svc.number_spatial_layers > 1) return; // In-place temporal filter. If psnr calculation is enabled, we store the
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index e8f752b..c87fe9e 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -2418,16 +2418,15 @@ av1_set_size_dependent_vars(cpi, &q, &bottom_index, &top_index); av1_set_mv_search_params(cpi); - if (cm->current_frame.frame_number == 0 && svc->number_temporal_layers > 1 && - svc->number_spatial_layers == 1) { + if (cm->current_frame.frame_number == 0 && cpi->ppi->use_svc) { const SequenceHeader *seq_params = cm->seq_params; if (aom_alloc_frame_buffer( - &cpi->svc.source_last_ref, cm->width, cm->height, - seq_params->subsampling_x, seq_params->subsampling_y, - seq_params->use_highbitdepth, cpi->oxcf.border_in_pixels, - cm->features.byte_alignment, 0)) { + &cpi->svc.source_last_TL0, cpi->oxcf.frm_dim_cfg.width, + cpi->oxcf.frm_dim_cfg.height, seq_params->subsampling_x, + seq_params->subsampling_y, seq_params->use_highbitdepth, + cpi->oxcf.border_in_pixels, cm->features.byte_alignment, 0)) { aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR, - "Failed to allocate buffer for source_last_ref"); + "Failed to allocate buffer for source_last_TL0"); } } @@ -2551,8 +2550,7 @@ // encoded at high/max QP, and if so, set the q and adjust some rate // control parameters. if (cpi->sf.rt_sf.overshoot_detection_cbr == FAST_DETECTION_MAXQ && - (cpi->rc.high_source_sad || - (cpi->ppi->use_svc && cpi->svc.high_source_sad_superframe))) { + cpi->rc.high_source_sad) { if (av1_encodedframe_overshoot_cbr(cpi, &q)) { av1_set_quantizer(cm, q_cfg->qm_minlevel, q_cfg->qm_maxlevel, q, q_cfg->enable_chroma_deltaq, q_cfg->enable_hdr_deltaq); @@ -2630,8 +2628,6 @@ sf->rt_sf.gf_refresh_based_on_qp) av1_adjust_gf_refresh_qp_one_pass_rt(cpi); - av1_svc_update_frame_number_buffslot(cpi); - #if CONFIG_COLLECT_COMPONENT_TIMING end_timing(cpi, av1_encode_frame_time); #endif
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h index 9333f7c..f4c345f 100644 --- a/av1/encoder/encoder_alloc.h +++ b/av1/encoder/encoder_alloc.h
@@ -269,7 +269,7 @@ aom_free_frame_buffer(&cpi->scaled_source); aom_free_frame_buffer(&cpi->scaled_last_source); aom_free_frame_buffer(&cpi->orig_source); - aom_free_frame_buffer(&cpi->svc.source_last_ref); + aom_free_frame_buffer(&cpi->svc.source_last_TL0); free_token_info(token_info);
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c index ab8d774..268b4b3 100644 --- a/av1/encoder/nonrd_pickmode.c +++ b/av1/encoder/nonrd_pickmode.c
@@ -2394,6 +2394,17 @@ use_golden_ref_frame = cpi->ref_frame_flags & AOM_GOLD_FLAG ? use_golden_ref_frame : 0; + // For spatial layers: enable golden ref if it is set by user and + // corresponds to the lower spatial layer. + if (cpi->svc.spatial_layer_id > 0 && (cpi->ref_frame_flags & AOM_GOLD_FLAG) && + x->content_state_sb.source_sad_nonrd < kHighSad) { + const int buffslot_golden = + cpi->ppi->rtc_ref.ref_idx[GOLDEN_FRAME - LAST_FRAME]; + if (cpi->svc.buffer_time_index[buffslot_golden] == + cpi->svc.current_superframe) + use_golden_ref_frame = 1; + } + use_ref_frame[ALTREF_FRAME] = use_alt_ref_frame; use_ref_frame[GOLDEN_FRAME] = use_golden_ref_frame; use_ref_frame[LAST_FRAME] = use_last_ref_frame; @@ -3340,8 +3351,9 @@ get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) continue; - // For screen content: - if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN) { + // For screen content: for base spatial layer only for now. + if (cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN && + cpi->svc.spatial_layer_id == 0) { // If source_sad is computed: skip non-zero motion // check for stationary (super)blocks. Otherwise if superblock // has motion skip the modes with zero motion for flat blocks, @@ -3445,6 +3457,7 @@ // skip newmv if the motion vector is (0, 0), and color is not set. if (this_mode == NEWMV && cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN && + cpi->svc.spatial_layer_id == 0 && cpi->sf.rt_sf.source_metrics_sb_nonrd) { if (frame_mv[this_mode][ref_frame].as_int == 0 && x->content_state_sb.source_sad_nonrd != kZeroSad &&
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index eb88dd0..05a92fe 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c
@@ -2758,9 +2758,14 @@ int last_src_ystride; int last_src_width; int last_src_height; - if (cm->spatial_layer_id != 0 || cm->width != cm->render_width || - cm->height != cm->render_height || unscaled_src == NULL || - unscaled_last_src == NULL) { + int width = cm->width; + int height = cm->height; + if (cpi->svc.number_spatial_layers > 1) { + width = cpi->oxcf.frm_dim_cfg.width; + height = cpi->oxcf.frm_dim_cfg.height; + } + if (width != cm->render_width || height != cm->render_height || + unscaled_src == NULL || unscaled_last_src == NULL) { if (cpi->src_sad_blk_64x64) { aom_free(cpi->src_sad_blk_64x64); cpi->src_sad_blk_64x64 = NULL; @@ -2787,8 +2792,12 @@ rc->max_block_source_sad = 0; rc->prev_avg_source_sad = rc->avg_source_sad; if (src_width == last_src_width && src_height == last_src_height) { - const int num_mi_cols = cm->mi_params.mi_cols; - const int num_mi_rows = cm->mi_params.mi_rows; + int num_mi_cols = cm->mi_params.mi_cols; + int num_mi_rows = cm->mi_params.mi_rows; + if (cpi->svc.number_spatial_layers > 1) { + num_mi_cols = cpi->svc.mi_cols_full_resoln; + num_mi_rows = cpi->svc.mi_rows_full_resoln; + } int num_zero_temp_sad = 0; uint32_t min_thresh = 10000; if (cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN) min_thresh = 100000; @@ -2810,8 +2819,7 @@ // Flag to check light change or not. const int check_light_change = 0; // Store blkwise SAD for later use - if ((cm->spatial_layer_id == 0) && (cm->width == cm->render_width) && - (cm->height == cm->render_height)) { + if (width == cm->render_width && height == cm->render_height) { if (cpi->src_sad_blk_64x64 == NULL) { CHECK_MEM_ERROR( cm, cpi->src_sad_blk_64x64, @@ -2869,7 +2877,22 @@ rc->percent_blocks_with_motion = ((num_samples - num_zero_temp_sad) * 100) / num_samples; } - cpi->svc.high_source_sad_superframe = rc->high_source_sad; + // Scene detection is only on base SLO, and using full/orignal resolution. + // Pass the state to the upper spatial layers. + if (cpi->svc.number_spatial_layers > 1) { + SVC *svc = &cpi->svc; + for (int sl = 0; sl < svc->number_spatial_layers; ++sl) { + int tl = svc->temporal_layer_id; + const int layer = LAYER_IDS_TO_IDX(sl, tl, svc->number_temporal_layers); + LAYER_CONTEXT *lc = &svc->layer_context[layer]; + RATE_CONTROL *lrc = &lc->rc; + lrc->high_source_sad = rc->high_source_sad; + lrc->frame_source_sad = rc->frame_source_sad; + lrc->avg_source_sad = rc->avg_source_sad; + lrc->percent_blocks_with_motion = rc->percent_blocks_with_motion; + lrc->max_block_source_sad = rc->max_block_source_sad; + } + } } /*!\brief Set the GF baseline interval for 1 pass real-time mode.
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c index aff79e3..41cf959 100644 --- a/av1/encoder/svc_layercontext.c +++ b/av1/encoder/svc_layercontext.c
@@ -209,6 +209,7 @@ cr->sb_index = lc->sb_index; cr->actual_num_seg1_blocks = lc->actual_num_seg1_blocks; cr->actual_num_seg2_blocks = lc->actual_num_seg2_blocks; + cr->counter_encode_maxq_scene_change = lc->counter_encode_maxq_scene_change; } svc->skip_mvsearch_last = 0; svc->skip_mvsearch_gf = 0; @@ -251,6 +252,7 @@ lc->sb_index = cr->sb_index; lc->actual_num_seg1_blocks = cr->actual_num_seg1_blocks; lc->actual_num_seg2_blocks = cr->actual_num_seg2_blocks; + lc->counter_encode_maxq_scene_change = cr->counter_encode_maxq_scene_change; } // For any buffer slot that is refreshed, update it with // the spatial_layer_id and the current_superframe. @@ -367,7 +369,10 @@ cpi->common.height = height; alloc_mb_mode_info_buffers(cpi); av1_update_frame_size(cpi); - if (svc->spatial_layer_id == 0) svc->high_source_sad_superframe = 0; + if (svc->spatial_layer_id == svc->number_spatial_layers - 1) { + svc->mi_cols_full_resoln = cpi->common.mi_params.mi_cols; + svc->mi_rows_full_resoln = cpi->common.mi_params.mi_rows; + } } enum { @@ -539,13 +544,28 @@ } } } -void av1_svc_update_frame_number_buffslot(AV1_COMP *const cpi) { - SVC *const svc = &cpi->svc; - const AV1_COMMON *const cm = &cpi->common; - const CurrentFrame *const current_frame = &cm->current_frame; - const RTC_REF *const rtc_ref = &cpi->ppi->rtc_ref; - for (int i = 0; i < 8; i++) { - if (current_frame->frame_type == KEY_FRAME || rtc_ref->refresh[i] == 1) - svc->frame_number_buffslot[i] = current_frame->frame_number; + +void av1_svc_set_last_source(AV1_COMP *const cpi, EncodeFrameInput *frame_input, + YV12_BUFFER_CONFIG *prev_source) { + if (cpi->svc.spatial_layer_id == 0) { + // For base spatial layer: if the LAST reference (index 0) is not + // the previous (super)frame set the last_source to the source corresponding + // to the last TL0, otherwise keep it at prev_source. + frame_input->last_source = prev_source != NULL ? prev_source : NULL; + if (cpi->svc.current_superframe > 0) { + const int buffslot_last = cpi->ppi->rtc_ref.ref_idx[0]; + if (cpi->svc.buffer_time_index[buffslot_last] < + cpi->svc.current_superframe - 1) + frame_input->last_source = &cpi->svc.source_last_TL0; + } + } else if (cpi->svc.spatial_layer_id > 0) { + // For spatial enhancement layers: the previous source (prev_source) + // corresponds to the lower spatial layer (which is the same source so + // we can't use that), so always set the last_source to the source of the + // last TL0. + if (cpi->svc.current_superframe > 0) + frame_input->last_source = &cpi->svc.source_last_TL0; + else + frame_input->last_source = NULL; } }
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h index d4b3d1a..2cec00f 100644 --- a/av1/encoder/svc_layercontext.h +++ b/av1/encoder/svc_layercontext.h
@@ -11,6 +11,7 @@ #ifndef AOM_AV1_ENCODER_SVC_LAYERCONTEXT_H_ #define AOM_AV1_ENCODER_SVC_LAYERCONTEXT_H_ +#include "aom_scale/yv12config.h" #include "av1/encoder/aq_cyclicrefresh.h" #include "av1/encoder/encoder.h" #include "av1/encoder/ratectrl.h" @@ -106,8 +107,9 @@ int temporal_layer_fb[REF_FRAMES]; int num_encoded_top_layer; int first_layer_denoise; - int high_source_sad_superframe; - YV12_BUFFER_CONFIG source_last_ref; + YV12_BUFFER_CONFIG source_last_TL0; + int mi_cols_full_resoln; + int mi_rows_full_resoln; /*!\endcond */ /*! @@ -135,14 +137,10 @@ * Force zero-mv in mode search for the spatial/inter-layer reference. */ int force_zero_mode_spatial_ref; - - /*! - * Frame numbers corresponding to each of the 8 reference buffer slots. - */ - unsigned int frame_number_buffslot[8]; } SVC; struct AV1_COMP; +struct EncodeFrameInput; /*!\brief Initialize layer context data from init_config(). * @@ -283,7 +281,9 @@ void av1_svc_check_reset_layer_rc_flag(struct AV1_COMP *const cpi); -void av1_svc_update_frame_number_buffslot(struct AV1_COMP *const cpi); +void av1_svc_set_last_source(struct AV1_COMP *const cpi, + struct EncodeFrameInput *frame_input, + YV12_BUFFER_CONFIG *prev_source); #ifdef __cplusplus } // extern "C"
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c index 63f8077..8d07311 100644 --- a/av1/encoder/var_based_part.c +++ b/av1/encoder/var_based_part.c
@@ -1337,7 +1337,7 @@ int variance4x4downsample[64]; const int segment_id = xd->mi[0]->segment_id; uint64_t blk_sad = 0; - if (cpi->src_sad_blk_64x64 != NULL) { + if (cpi->src_sad_blk_64x64 != NULL && cpi->svc.number_spatial_layers == 1) { const int sb_size_by_mb = (cm->seq_params->sb_size == BLOCK_128X128) ? (cm->seq_params->mib_size >> 1) : cm->seq_params->mib_size;
diff --git a/test/svc_datarate_test.cc b/test/svc_datarate_test.cc index c3023ad..aa9f1b4 100644 --- a/test/svc_datarate_test.cc +++ b/test/svc_datarate_test.cc
@@ -672,6 +672,39 @@ // This means 30 = #frames(60) - #TL2_frames(30). EXPECT_EQ((int)GetMismatchFrames(), 30); } + + virtual void BasicRateTargetingSVC1TL3SLScreenTest() { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 0; + + ::libaom_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); + + const int bitrate_array[2] = { 800, 1200 }; + cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; + ResetModel(); + screen_mode_ = 1; + number_temporal_layers_ = 1; + number_spatial_layers_ = 3; + target_layer_bitrate_[0] = 30 * cfg_.rc_target_bitrate / 100; + target_layer_bitrate_[1] = 60 * cfg_.rc_target_bitrate / 100; + target_layer_bitrate_[2] = cfg_.rc_target_bitrate; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + for (int i = 0; i < number_temporal_layers_ * number_spatial_layers_; i++) { + ASSERT_GE(effective_datarate_tl[i], target_layer_bitrate_[i] * 0.50) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_tl[i], target_layer_bitrate_[i] * 1.5) + << " The datarate for the file is greater than target by too much!"; + } + EXPECT_EQ((int)GetMismatchFrames(), 0); + } + virtual void BasicRateTargetingSVC1TL1SLScreenScCutsMotionTest() { cfg_.rc_buf_initial_sz = 500; cfg_.rc_buf_optimal_sz = 500; @@ -1839,6 +1872,12 @@ BasicRateTargetingSVC3TL1SLScreenTest(); } +// Check basic rate targeting for CBR, for 3 spatial layers, 1 temporal +// for screen mode. +TEST_P(DatarateTestSVC, BasicRateTargetingSVC1TL3SLScreen) { + BasicRateTargetingSVC1TL3SLScreenTest(); +} + // Check basic rate targeting for CBR, for 1 temporal layer, 1 spatial // for screen mode, with source with many scene cuts and motion. TEST_P(DatarateTestSVC, BasicRateTargetingSVC1TL1SLScreenScCutsMotion) {