rtc: Layered CBR rate control for SVC Implement layer context based CBR rate control, to allow each layer to target its own layer bitrate. Added datarate unittest for 3 temporal layers. Temporal layers can be tested with the sample encoder: svc_encoder_rtc Change-Id: I5892c0998a81ac82cea8b2035197627f6cd32d19
diff --git a/av1/av1.cmake b/av1/av1.cmake index d0e6278..0fe5bc4 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -203,6 +203,8 @@ "${AOM_ROOT}/av1/encoder/segmentation.h" "${AOM_ROOT}/av1/encoder/speed_features.c" "${AOM_ROOT}/av1/encoder/speed_features.h" + "${AOM_ROOT}/av1/encoder/svc_layercontext.c" + "${AOM_ROOT}/av1/encoder/svc_layercontext.h" "${AOM_ROOT}/av1/encoder/temporal_filter.c" "${AOM_ROOT}/av1/encoder/temporal_filter.h" "${AOM_ROOT}/av1/encoder/tokenize.c"
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index 942650d..f1c8515 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -2162,6 +2162,8 @@ aom_svc_layer_id_t *const data = va_arg(args, aom_svc_layer_id_t *); ctx->cpi->common.spatial_layer_id = data->spatial_layer_id; ctx->cpi->common.temporal_layer_id = data->temporal_layer_id; + ctx->cpi->svc.spatial_layer_id = data->spatial_layer_id; + ctx->cpi->svc.temporal_layer_id = data->temporal_layer_id; return AOM_CODEC_OK; } @@ -2171,6 +2173,8 @@ aom_svc_params_t *const params = va_arg(args, aom_svc_params_t *); cpi->common.number_spatial_layers = params->number_spatial_layers; cpi->common.number_temporal_layers = params->number_temporal_layers; + cpi->svc.number_spatial_layers = params->number_spatial_layers; + cpi->svc.number_temporal_layers = params->number_temporal_layers; if (cpi->common.number_spatial_layers > 1 || cpi->common.number_temporal_layers > 1) { unsigned int sl, tl; @@ -2184,10 +2188,14 @@ lc->min_q = params->min_quantizers[layer]; lc->scaling_factor_num = params->scaling_factor_num[sl]; lc->scaling_factor_den = params->scaling_factor_den[sl]; - lc->layer_target_bitrate = params->layer_target_bitrate[layer]; + lc->layer_target_bitrate = 1000 * params->layer_target_bitrate[layer]; lc->framerate_factor = params->framerate_factor[tl]; } } + if (cpi->common.current_frame.frame_number == 0) + av1_init_layer_context(cpi); + else + av1_update_layer_context_change_config(cpi, cpi->oxcf.target_bandwidth); } return AOM_CODEC_OK; }
diff --git a/av1/encoder/aq_cyclicrefresh.c b/av1/encoder/aq_cyclicrefresh.c index 9c57c49..cd68425 100644 --- a/av1/encoder/aq_cyclicrefresh.c +++ b/av1/encoder/aq_cyclicrefresh.c
@@ -19,46 +19,6 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_ports/system_state.h" -struct CYCLIC_REFRESH { - // Percentage of blocks per frame that are targeted as candidates - // for cyclic refresh. - int percent_refresh; - // Maximum q-delta as percentage of base q. - int max_qdelta_perc; - // Superblock starting index for cycling through the frame. - int sb_index; - // Controls how long block will need to wait to be refreshed again, in - // excess of the cycle time, i.e., in the case of all zero motion, block - // will be refreshed every (100/percent_refresh + time_for_refresh) frames. - int time_for_refresh; - // Target number of (4x4) blocks that are set for delta-q. - int target_num_seg_blocks; - // Actual number of (4x4) blocks that were applied delta-q. - int actual_num_seg1_blocks; - int actual_num_seg2_blocks; - // RD mult. parameters for segment 1. - int rdmult; - // Cyclic refresh map. - int8_t *map; - // Map of the last q a block was coded at. - uint8_t *last_coded_q_map; - // Thresholds applied to the projected rate/distortion of the coding block, - // when deciding whether block should be refreshed. - int64_t thresh_rate_sb; - int64_t thresh_dist_sb; - // Threshold applied to the motion vector (in units of 1/8 pel) of the - // coding block, when deciding whether block should be refreshed. - int16_t motion_thresh; - // Rate target ratio to set q delta. - double rate_ratio_qdelta; - // Boost factor for rate target ratio, for segment CR_SEGMENT_ID_BOOST2. - int rate_boost_fac; - double low_content_avg; - int qindex_delta[3]; - double weight_segment; - int apply_cyclic_refresh; -}; - CYCLIC_REFRESH *av1_cyclic_refresh_alloc(int mi_rows, int mi_cols) { size_t last_coded_q_map_size; CYCLIC_REFRESH *const cr = aom_calloc(1, sizeof(*cr)); @@ -365,6 +325,7 @@ int qp_thresh = AOMMIN(20, rc->best_quality << 1); cr->apply_cyclic_refresh = 1; if (frame_is_intra_only(cm) || is_lossless_requested(&cpi->oxcf) || + cpi->svc.temporal_layer_id > 0 || rc->avg_frame_qindex[INTER_FRAME] < qp_thresh) { cr->apply_cyclic_refresh = 0; return;
diff --git a/av1/encoder/aq_cyclicrefresh.h b/av1/encoder/aq_cyclicrefresh.h index ddabae6..774b165 100644 --- a/av1/encoder/aq_cyclicrefresh.h +++ b/av1/encoder/aq_cyclicrefresh.h
@@ -27,9 +27,48 @@ // Maximum rate target ratio for setting segment delta-qp. #define CR_MAX_RATE_TARGET_RATIO 4.0 +struct CYCLIC_REFRESH { + // Percentage of blocks per frame that are targeted as candidates + // for cyclic refresh. + int percent_refresh; + // Maximum q-delta as percentage of base q. + int max_qdelta_perc; + // Superblock starting index for cycling through the frame. + int sb_index; + // Controls how long block will need to wait to be refreshed again, in + // excess of the cycle time, i.e., in the case of all zero motion, block + // will be refreshed every (100/percent_refresh + time_for_refresh) frames. + int time_for_refresh; + // Target number of (4x4) blocks that are set for delta-q. + int target_num_seg_blocks; + // Actual number of (4x4) blocks that were applied delta-q. + int actual_num_seg1_blocks; + int actual_num_seg2_blocks; + // RD mult. parameters for segment 1. + int rdmult; + // Cyclic refresh map. + int8_t *map; + // Map of the last q a block was coded at. + uint8_t *last_coded_q_map; + // Thresholds applied to the projected rate/distortion of the coding block, + // when deciding whether block should be refreshed. + int64_t thresh_rate_sb; + int64_t thresh_dist_sb; + // Threshold applied to the motion vector (in units of 1/8 pel) of the + // coding block, when deciding whether block should be refreshed. + int16_t motion_thresh; + // Rate target ratio to set q delta. + double rate_ratio_qdelta; + // Boost factor for rate target ratio, for segment CR_SEGMENT_ID_BOOST2. + int rate_boost_fac; + double low_content_avg; + int qindex_delta[3]; + double weight_segment; + int apply_cyclic_refresh; +}; + struct AV1_COMP; -struct CYCLIC_REFRESH; typedef struct CYCLIC_REFRESH CYCLIC_REFRESH; CYCLIC_REFRESH *av1_cyclic_refresh_alloc(int mi_rows, int mi_cols);
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index 12dd2aa..06e575f 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -357,7 +357,7 @@ const int intra_only = frame_params->frame_type == KEY_FRAME || frame_params->frame_type == INTRA_ONLY_FRAME; - if (intra_only || frame_params->error_resilient_mode || + if (intra_only || frame_params->error_resilient_mode || cpi->use_svc || cpi->ext_use_primary_ref_none) { return PRIMARY_REF_NONE; } @@ -1459,5 +1459,7 @@ cpi->droppable = is_frame_droppable(cpi); } + if (cpi->use_svc) av1_save_layer_context(cpi); + return AOM_CODEC_OK; }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index 38011d6..059ff57 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -2609,6 +2609,9 @@ cm->number_spatial_layers > 1 ? cm->number_spatial_layers - 1 : 0; init_seq_coding_tools(&cm->seq_params, cm, oxcf, cpi->use_svc); } + + if (cpi->use_svc) + av1_update_layer_context_change_config(cpi, oxcf->target_bandwidth); } AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf,
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index cb99d57..de153e2 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -39,6 +39,7 @@ #include "av1/encoder/ratectrl.h" #include "av1/encoder/rd.h" #include "av1/encoder/speed_features.h" +#include "av1/encoder/svc_layercontext.h" #include "av1/encoder/tokenize.h" #include "av1/encoder/block.h" @@ -164,25 +165,6 @@ SS_CFG_TOTAL = 2 } UENUM1BYTE(SS_CFG_OFFSET); -typedef struct { - int max_q; - int min_q; - int framerate_factor; - int layer_target_bitrate; - int scaling_factor_num; - int scaling_factor_den; - RATE_CONTROL rc; -} LAYER_CONTEXT; - -typedef struct SVC { - int external_ref_frame_config; - int non_reference_frame; - // Layer context used for rate control in one pass temporal CBR mode. - LAYER_CONTEXT layer_context[AOM_MAX_LAYERS]; - int ref_idx[INTER_REFS_PER_FRAME]; - int refresh[REF_FRAMES]; -} SVC; - #define MAX_LENGTH_TPL_FRAME_STATS (27 + 9) typedef struct TplDepStats {
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index b2505ee..b286227 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c
@@ -215,6 +215,24 @@ return target; } +// Update the buffer level for higher temporal layers, given the encoded current +// temporal layer. +static void update_layer_buffer_level(SVC *svc, int encoded_frame_size) { + const int current_temporal_layer = svc->temporal_layer_id; + for (int i = current_temporal_layer + 1; i < svc->number_temporal_layers; + ++i) { + const int layer = + LAYER_IDS_TO_IDX(svc->spatial_layer_id, i, svc->number_temporal_layers); + LAYER_CONTEXT *lc = &svc->layer_context[layer]; + RATE_CONTROL *lrc = &lc->rc; + lrc->bits_off_target += + (int)(lc->target_bandwidth / lc->framerate) - encoded_frame_size; + // Clip buffer level to maximum buffer size for the layer. + lrc->bits_off_target = + AOMMIN(lrc->bits_off_target, lrc->maximum_buffer_size); + lrc->buffer_level = lrc->bits_off_target; + } +} // Update the buffer level: leaky bucket model. static void update_buffer_level(AV1_COMP *cpi, int encoded_frame_size) { const AV1_COMMON *const cm = &cpi->common; @@ -229,6 +247,8 @@ // Clip the buffer level to the maximum specified buffer size. rc->bits_off_target = AOMMIN(rc->bits_off_target, rc->maximum_buffer_size); rc->buffer_level = rc->bits_off_target; + + if (cpi->use_svc) update_layer_buffer_level(&cpi->svc, encoded_frame_size); } int av1_rc_get_default_min_gf_interval(int width, int height, @@ -378,7 +398,7 @@ rcf = rc->rate_correction_factors[rf_lvl]; } else { if ((cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame) && - !rc->is_src_frame_alt_ref && + !rc->is_src_frame_alt_ref && !cpi->use_svc && (cpi->oxcf.rc_mode != AOM_CBR || cpi->oxcf.gf_cbr_boost_pct > 20)) rcf = rc->rate_correction_factors[GF_ARF_STD]; else @@ -404,7 +424,7 @@ rc->rate_correction_factors[rf_lvl] = factor; } else { if ((cpi->refresh_alt_ref_frame || cpi->refresh_golden_frame) && - !rc->is_src_frame_alt_ref && + !rc->is_src_frame_alt_ref && !cpi->use_svc && (cpi->oxcf.rc_mode != AOM_CBR || cpi->oxcf.gf_cbr_boost_pct > 20)) rc->rate_correction_factors[GF_ARF_STD] = factor; else @@ -565,7 +585,7 @@ // In CBR mode, this makes sure q is between oscillating Qs to prevent // resonance. - if (cpi->oxcf.rc_mode == AOM_CBR && + if (cpi->oxcf.rc_mode == AOM_CBR && !cpi->use_svc && (cpi->rc.rc_1_frame * cpi->rc.rc_2_frame == -1) && cpi->rc.q_1_frame != cpi->rc.q_2_frame) { q = clamp(q, AOMMIN(cpi->rc.q_1_frame, cpi->rc.q_2_frame), @@ -1783,7 +1803,17 @@ } else { target = rc->avg_frame_bandwidth; } - + if (cpi->use_svc) { + // Note that for layers, avg_frame_bandwidth is the cumulative + // per-frame-bandwidth. For the target size of this frame, use the + // layer average frame size (i.e., non-cumulative per-frame-bw). + int layer = + LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id, cpi->svc.temporal_layer_id, + cpi->svc.number_temporal_layers); + const LAYER_CONTEXT *lc = &cpi->svc.layer_context[layer]; + target = lc->avg_frame_size; + min_frame_target = AOMMAX(lc->avg_frame_size >> 4, FRAME_OVERHEAD_BITS); + } if (diff > 0) { // Lower the target bandwidth for this frame. const int pct_low = (int)AOMMIN(diff / one_pct_bits, oxcf->under_shoot_pct); @@ -1832,6 +1862,10 @@ AV1_COMMON *const cm = &cpi->common; FRAME_UPDATE_TYPE frame_update_type; int target; + if (cpi->use_svc) { + av1_update_temporal_layer_framerate(cpi); + av1_restore_layer_context(cpi); + } if (rc->frames_to_key == 0 || (frame_flags & FRAMEFLAGS_KEY)) { frame_params->frame_type = KEY_FRAME; rc->this_key_frame_forced = @@ -1840,11 +1874,13 @@ rc->kf_boost = DEFAULT_KF_BOOST_RT; rc->source_alt_ref_active = 0; frame_update_type = KF_UPDATE; + if (cpi->use_svc && cm->current_frame.frame_number > 0) + av1_svc_reset_temporal_layers(cpi, 1); } else { frame_params->frame_type = INTER_FRAME; frame_update_type = LF_UPDATE; } - if (rc->frames_till_gf_update_due == 0) { + if (rc->frames_till_gf_update_due == 0 && cpi->svc.temporal_layer_id == 0) { GF_GROUP *const gf_group = &cpi->gf_group; if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) av1_cyclic_refresh_set_golden_update(cpi); @@ -1852,10 +1888,17 @@ rc->baseline_gf_interval = MAX_GF_INTERVAL; if (rc->baseline_gf_interval > rc->frames_to_key) rc->baseline_gf_interval = rc->frames_to_key; - rc->frames_till_gf_update_due = rc->baseline_gf_interval; rc->gfu_boost = DEFAULT_GF_BOOST_RT; rc->constrained_gf_group = (rc->baseline_gf_interval >= rc->frames_to_key) ? 1 : 0; + // SVC does not use GF as periodid boost. + // TODO(marpan): Find better way to disable this for SVC. + if (cpi->use_svc) { + rc->baseline_gf_interval = MAX_STATIC_GF_GROUP_LENGTH; + rc->gfu_boost = 1; + rc->constrained_gf_group = 0; + } + rc->frames_till_gf_update_due = rc->baseline_gf_interval; frame_update_type = GF_UPDATE; gf_group->index = 0; gf_group->size = rc->baseline_gf_interval;
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c new file mode 100644 index 0000000..6851422 --- /dev/null +++ b/av1/encoder/svc_layercontext.c
@@ -0,0 +1,226 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <math.h> + +#include "av1/encoder/encoder.h" + +static void swap_ptr(void *a, void *b) { + void **a_p = (void **)a; + void **b_p = (void **)b; + void *c = *a_p; + *a_p = *b_p; + *b_p = c; +} + +void av1_init_layer_context(AV1_COMP *const cpi) { + AV1_COMMON *const cm = &cpi->common; + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + SVC *const svc = &cpi->svc; + int mi_rows = cpi->common.mi_rows; + int mi_cols = cpi->common.mi_cols; + + for (int sl = 0; sl < svc->number_spatial_layers; ++sl) { + for (int tl = 0; tl < svc->number_temporal_layers; ++tl) { + int layer = LAYER_IDS_TO_IDX(sl, tl, svc->number_temporal_layers); + LAYER_CONTEXT *const lc = &svc->layer_context[layer]; + RATE_CONTROL *const lrc = &lc->rc; + lrc->ni_av_qi = oxcf->worst_allowed_q; + lrc->total_actual_bits = 0; + lrc->total_target_vs_actual = 0; + lrc->ni_tot_qi = 0; + lrc->tot_q = 0.0; + lrc->avg_q = 0.0; + lrc->ni_frames = 0; + lrc->decimation_count = 0; + lrc->decimation_factor = 0; + lrc->worst_quality = av1_quantizer_to_qindex(lc->max_q); + lrc->best_quality = av1_quantizer_to_qindex(lc->min_q); + for (int i = 0; i < RATE_FACTOR_LEVELS; ++i) { + lrc->rate_correction_factors[i] = 1.0; + } + lc->target_bandwidth = lc->layer_target_bitrate; + lrc->last_q[INTER_FRAME] = lrc->worst_quality; + lrc->avg_frame_qindex[INTER_FRAME] = lrc->worst_quality; + lrc->avg_frame_qindex[KEY_FRAME] = lrc->worst_quality; + lrc->buffer_level = + oxcf->starting_buffer_level_ms * lc->target_bandwidth / 1000; + lrc->bits_off_target = lrc->buffer_level; + // Initialize the cyclic refresh parameters. If spatial layers are used + // (i.e., ss_number_layers > 1), these need to be updated per spatial + // layer. Cyclic refresh is only applied on base temporal layer. + if (svc->number_spatial_layers > 1 && tl == 0) { + size_t last_coded_q_map_size; + lc->sb_index = 0; + lc->actual_num_seg1_blocks = 0; + lc->actual_num_seg2_blocks = 0; + lc->counter_encode_maxq_scene_change = 0; + CHECK_MEM_ERROR(cm, lc->map, + aom_malloc(mi_rows * mi_cols * sizeof(*lc->map))); + memset(lc->map, 0, mi_rows * mi_cols); + last_coded_q_map_size = + mi_rows * mi_cols * sizeof(*lc->last_coded_q_map); + CHECK_MEM_ERROR(cm, lc->last_coded_q_map, + aom_malloc(last_coded_q_map_size)); + assert(MAXQ <= 255); + memset(lc->last_coded_q_map, MAXQ, last_coded_q_map_size); + } + } + } +} + +// Update the layer context from a change_config() call. +void av1_update_layer_context_change_config(AV1_COMP *const cpi, + const int target_bandwidth) { + const RATE_CONTROL *const rc = &cpi->rc; + SVC *const svc = &cpi->svc; + int layer = 0; + int spatial_layer_target = 0; + float bitrate_alloc = 1.0; + + for (int sl = 0; sl < svc->number_spatial_layers; ++sl) { + for (int tl = 0; tl < svc->number_temporal_layers; ++tl) { + layer = LAYER_IDS_TO_IDX(sl, tl, svc->number_temporal_layers); + LAYER_CONTEXT *const lc = &svc->layer_context[layer]; + svc->layer_context[layer].target_bandwidth = lc->layer_target_bitrate; + } + spatial_layer_target = svc->layer_context[layer].target_bandwidth; + for (int tl = 0; tl < svc->number_temporal_layers; ++tl) { + LAYER_CONTEXT *const lc = + &svc->layer_context[sl * svc->number_temporal_layers + tl]; + RATE_CONTROL *const lrc = &lc->rc; + lc->spatial_layer_target_bandwidth = spatial_layer_target; + bitrate_alloc = (float)lc->target_bandwidth / target_bandwidth; + lrc->starting_buffer_level = + (int64_t)(rc->starting_buffer_level * bitrate_alloc); + lrc->optimal_buffer_level = + (int64_t)(rc->optimal_buffer_level * bitrate_alloc); + lrc->maximum_buffer_size = + (int64_t)(rc->maximum_buffer_size * bitrate_alloc); + lrc->bits_off_target = + AOMMIN(lrc->bits_off_target, lrc->maximum_buffer_size); + lrc->buffer_level = AOMMIN(lrc->buffer_level, lrc->maximum_buffer_size); + lc->framerate = cpi->framerate / lc->framerate_factor; + lrc->avg_frame_bandwidth = (int)(lc->target_bandwidth / lc->framerate); + lrc->max_frame_bandwidth = rc->max_frame_bandwidth; + lrc->worst_quality = av1_quantizer_to_qindex(lc->max_q); + lrc->best_quality = av1_quantizer_to_qindex(lc->min_q); + } + } +} + +static LAYER_CONTEXT *get_layer_context(AV1_COMP *const cpi) { + return &cpi->svc.layer_context[cpi->svc.spatial_layer_id * + cpi->svc.number_temporal_layers + + cpi->svc.temporal_layer_id]; +} + +void av1_update_temporal_layer_framerate(AV1_COMP *const cpi) { + SVC *const svc = &cpi->svc; + LAYER_CONTEXT *const lc = get_layer_context(cpi); + RATE_CONTROL *const lrc = &lc->rc; + const int tl = svc->temporal_layer_id; + lc->framerate = cpi->framerate / lc->framerate_factor; + lrc->avg_frame_bandwidth = (int)(lc->target_bandwidth / lc->framerate); + lrc->max_frame_bandwidth = cpi->rc.max_frame_bandwidth; + // Update the average layer frame size (non-cumulative per-frame-bw). + if (tl == 0) { + lc->avg_frame_size = lrc->avg_frame_bandwidth; + } else { + int prev_layer = svc->spatial_layer_id * svc->number_temporal_layers + + svc->temporal_layer_id - 1; + LAYER_CONTEXT *const lcprev = &svc->layer_context[prev_layer]; + const double prev_layer_framerate = + cpi->framerate / lcprev->framerate_factor; + const int prev_layer_target_bandwidth = lcprev->layer_target_bitrate; + lc->avg_frame_size = + (int)((lc->target_bandwidth - prev_layer_target_bandwidth) / + (lc->framerate - prev_layer_framerate)); + } +} + +void av1_restore_layer_context(AV1_COMP *const cpi) { + LAYER_CONTEXT *const lc = get_layer_context(cpi); + const int old_frame_since_key = cpi->rc.frames_since_key; + const int old_frame_to_key = cpi->rc.frames_to_key; + // Restore layer rate control. + cpi->rc = lc->rc; + cpi->oxcf.target_bandwidth = lc->target_bandwidth; + // Reset the frames_since_key and frames_to_key counters to their values + // before the layer restore. Keep these defined for the stream (not layer). + cpi->rc.frames_since_key = old_frame_since_key; + cpi->rc.frames_to_key = old_frame_to_key; + // For spatial-svc, allow cyclic-refresh to be applied on the spatial layers, + // for the base temporal layer. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && + cpi->svc.number_spatial_layers > 1 && cpi->svc.temporal_layer_id == 0) { + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + swap_ptr(&cr->map, &lc->map); + swap_ptr(&cr->last_coded_q_map, &lc->last_coded_q_map); + cr->sb_index = lc->sb_index; + cr->actual_num_seg1_blocks = lc->actual_num_seg1_blocks; + cr->actual_num_seg2_blocks = lc->actual_num_seg2_blocks; + } +} + +void av1_save_layer_context(AV1_COMP *const cpi) { + SVC *const svc = &cpi->svc; + LAYER_CONTEXT *lc = get_layer_context(cpi); + // Reset gf counters on non-base temporal layer. + // TODO(marpan): Temporary for now, fix this. + if (svc->temporal_layer_id > 0) { + cpi->gf_group.index--; + if (cpi->rc.frames_till_gf_update_due > 0) + cpi->rc.frames_till_gf_update_due++; + } + lc->rc = cpi->rc; + lc->target_bandwidth = (int)cpi->oxcf.target_bandwidth; + // For spatial-svc, allow cyclic-refresh to be applied on the spatial layers, + // for the base temporal layer. + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && + cpi->svc.number_spatial_layers > 1 && cpi->svc.temporal_layer_id == 0) { + CYCLIC_REFRESH *const cr = cpi->cyclic_refresh; + signed char *temp = lc->map; + uint8_t *temp2 = lc->last_coded_q_map; + lc->map = cr->map; + cr->map = temp; + lc->last_coded_q_map = cr->last_coded_q_map; + cr->last_coded_q_map = temp2; + lc->sb_index = cr->sb_index; + lc->actual_num_seg1_blocks = cr->actual_num_seg1_blocks; + lc->actual_num_seg2_blocks = cr->actual_num_seg2_blocks; + } +} + +void av1_free_svc_cyclic_refresh(AV1_COMP *const cpi) { + SVC *const svc = &cpi->svc; + for (int sl = 0; sl < svc->number_spatial_layers; ++sl) { + for (int tl = 0; tl < svc->number_temporal_layers; ++tl) { + int layer = LAYER_IDS_TO_IDX(sl, tl, svc->number_temporal_layers); + LAYER_CONTEXT *const lc = &svc->layer_context[layer]; + if (lc->map) aom_free(lc->map); + if (lc->last_coded_q_map) aom_free(lc->last_coded_q_map); + } + } +} + +// Reset on key frame: reset counters, references and buffer updates. +void av1_svc_reset_temporal_layers(AV1_COMP *const cpi, int is_key) { + SVC *const svc = &cpi->svc; + LAYER_CONTEXT *lc = NULL; + for (int sl = 0; sl < svc->number_spatial_layers; ++sl) { + for (int tl = 0; tl < svc->number_temporal_layers; ++tl) { + lc = &cpi->svc.layer_context[sl * svc->number_temporal_layers + tl]; + if (is_key) lc->frames_from_key_frame = 0; + } + } + av1_update_temporal_layer_framerate(cpi); + av1_restore_layer_context(cpi); +}
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h new file mode 100644 index 0000000..01d3fdc --- /dev/null +++ b/av1/encoder/svc_layercontext.h
@@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019, Alliance for Open Media. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AOM_AV1_ENCODER_SVC_LAYERCONTEXT_H_ +#define AOM_AV1_ENCODER_SVC_LAYERCONTEXT_H_ + +#include "av1/encoder/aq_cyclicrefresh.h" +#include "av1/encoder/encoder.h" +#include "av1/encoder/ratectrl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + RATE_CONTROL rc; + int framerate_factor; + int layer_target_bitrate; + int scaling_factor_num; + int scaling_factor_den; + int target_bandwidth; + int spatial_layer_target_bandwidth; + double framerate; + int avg_frame_size; + int max_q; + int min_q; + int frames_from_key_frame; + // Cyclic refresh parameters (aq-mode=3), that need to be updated per-frame. + int sb_index; + int8_t *map; + uint8_t *last_coded_q_map; + int actual_num_seg1_blocks; + int actual_num_seg2_blocks; + int counter_encode_maxq_scene_change; + uint8_t speed; +} LAYER_CONTEXT; + +typedef struct SVC { + int spatial_layer_id; + int temporal_layer_id; + int number_spatial_layers; + int number_temporal_layers; + int external_ref_frame_config; + int non_reference_frame; + int ref_idx[INTER_REFS_PER_FRAME]; + int refresh[REF_FRAMES]; + // Layer context used for rate control in one pass temporal CBR mode or + // two pass spatial mode. + LAYER_CONTEXT layer_context[AOM_MAX_LAYERS]; +} SVC; + +struct AV1_COMP; + +// Initialize layer context data from init_config(). +void av1_init_layer_context(struct AV1_COMP *const cpi); + +// Update the layer context from a change_config() call. +void av1_update_layer_context_change_config(struct AV1_COMP *const cpi, + const int target_bandwidth); + +// Prior to encoding the frame, update framerate-related quantities +// for the current temporal layer. +void av1_update_temporal_layer_framerate(struct AV1_COMP *const cpi); + +// Prior to encoding the frame, set the layer context, for the current layer +// to be encoded, to the cpi struct. +void av1_restore_layer_context(struct AV1_COMP *const cpi); + +// Save the layer context after encoding the frame. +void av1_save_layer_context(struct AV1_COMP *const cpi); + +void av1_free_svc_cyclic_refresh(struct AV1_COMP *const cpi); + +void av1_svc_reset_temporal_layers(struct AV1_COMP *const cpi, int is_key); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // AOM_AV1_ENCODER_SVC_LAYERCONTEXT_H_
diff --git a/test/datarate_test.cc b/test/datarate_test.cc index d2bd161..bdaedfb 100644 --- a/test/datarate_test.cc +++ b/test/datarate_test.cc
@@ -52,6 +52,79 @@ bits_total_ = 0; denoiser_offon_test_ = 0; denoiser_offon_period_ = -1; + number_temporal_layers_ = 1; + for (int i = 0; i < 3; i++) { + target_layer_bitrate_[i] = 0; + effective_datarate_tl[i] = 0.0; + } + memset(&layer_id_, 0, sizeof(aom_svc_layer_id_t)); + memset(&svc_params_, 0, sizeof(aom_svc_params_t)); + memset(&ref_frame_config_, 0, sizeof(aom_svc_ref_frame_config_t)); + } + + // Layer pattern configuration. + int set_layer_pattern(int frame_cnt, aom_svc_layer_id_t *layer_id, + aom_svc_ref_frame_config_t *ref_frame_config) { + // No spatial layers in this test. + layer_id->spatial_layer_id = 0; + // Set the referende map buffer idx for the 7 references: + // LAST_FRAME (0), LAST2_FRAME(1), LAST3_FRAME(2), GOLDEN_FRAME(3), + // BWDREF_FRAME(4), ALTREF2_FRAME(5), ALTREF_FRAME(6). + for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = i; + for (int i = 0; i < 8; i++) ref_frame_config->refresh[i] = 0; + // Note only use LAST and GF for prediction in non-rd mode (speed 8). + int layer_flags = AOM_EFLAG_NO_REF_LAST2 | AOM_EFLAG_NO_REF_LAST3 | + AOM_EFLAG_NO_REF_ARF | AOM_EFLAG_NO_REF_BWD | + AOM_EFLAG_NO_REF_ARF2; + // 3-layer: + // 1 3 5 7 + // 2 6 + // 0 4 8 + if (frame_cnt % 4 == 0) { + // Base layer. + layer_id->temporal_layer_id = 0; + // Update LAST on layer 0, reference LAST and GF. + ref_frame_config->refresh[0] = 1; + } else if ((frame_cnt - 1) % 4 == 0) { + layer_id->temporal_layer_id = 2; + // First top layer: no updates, only reference LAST (TL0). + layer_flags |= AOM_EFLAG_NO_REF_GF; + } else if ((frame_cnt - 2) % 4 == 0) { + layer_id->temporal_layer_id = 1; + // Middle layer (TL1): update LAST2, only reference LAST (TL0). + ref_frame_config->refresh[1] = 1; + layer_flags |= AOM_EFLAG_NO_REF_GF; + } else if ((frame_cnt - 3) % 4 == 0) { + layer_id->temporal_layer_id = 2; + // Second top layer: no updates, only reference LAST. + // Set buffer idx for LAST to slot 1, since that was the slot + // updated in previous frame. So LAST is TL1 frame. + ref_frame_config->ref_idx[0] = 1; + ref_frame_config->ref_idx[1] = 0; + layer_flags |= AOM_EFLAG_NO_REF_GF; + } + return layer_flags; + } + + void initialize_svc(int number_temporal_layers, aom_svc_params *svc_params) { + svc_params->number_spatial_layers = 1; + svc_params->scaling_factor_num[0] = 1; + svc_params->scaling_factor_den[0] = 1; + svc_params->number_temporal_layers = number_temporal_layers; + for (int i = 0; i < number_temporal_layers; ++i) { + svc_params->max_quantizers[i] = 56; + svc_params->min_quantizers[i] = 2; + svc_params->layer_target_bitrate[i] = target_layer_bitrate_[i]; + } + svc_params->framerate_factor[0] = 1; + if (number_temporal_layers == 2) { + svc_params->framerate_factor[0] = 2; + svc_params->framerate_factor[1] = 1; + } else if (number_temporal_layers == 3) { + svc_params->framerate_factor[0] = 4; + svc_params->framerate_factor[1] = 2; + svc_params->framerate_factor[2] = 1; + } } virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video, @@ -59,6 +132,10 @@ if (video->frame() == 0) { encoder->Control(AOME_SET_CPUUSED, set_cpu_used_); encoder->Control(AV1E_SET_AQ_MODE, aq_mode_); + if (number_temporal_layers_ > 1) { + initialize_svc(number_temporal_layers_, &svc_params_); + encoder->Control(AV1E_SET_SVC_PARAMS, &svc_params_); + } } if (denoiser_offon_test_) { @@ -72,6 +149,15 @@ encoder->Control(AV1E_SET_NOISE_SENSITIVITY, denoiser_on_); + if (number_temporal_layers_ > 1) { + // Set the reference/update flags, layer_id, and reference_map + // buffer index. + frame_flags_ = + set_layer_pattern(video->frame(), &layer_id_, &ref_frame_config_); + encoder->Control(AV1E_SET_SVC_LAYER_ID, &layer_id_); + encoder->Control(AV1E_SET_SVC_REF_FRAME_CONFIG, &ref_frame_config_); + } + const aom_rational_t tb = video->timebase(); timebase_ = static_cast<double>(tb.num) / tb.den; duration_ = 0; @@ -104,6 +190,12 @@ // Update the total encoded bits. bits_total_ += frame_size_in_bits; + if (number_temporal_layers_ > 1) { + // Update the layer cumulative bitrate. + for (int i = layer_id_.temporal_layer_id; i < number_temporal_layers_; + i++) + effective_datarate_tl[i] += 1.0 * frame_size_in_bits; + } // Update the most recent pts. last_pts_ = pkt->data.frame.pts; ++frame_number_; @@ -114,6 +206,11 @@ duration_ = (last_pts_ + 1) * timebase_; // Effective file datarate: effective_datarate_ = (bits_total_ / 1000.0) / duration_; + if (number_temporal_layers_ > 1) { + for (int i = 0; i < number_temporal_layers_; i++) + effective_datarate_tl[i] = + (effective_datarate_tl[i] / 1000) / duration_; + } } virtual void BasicRateTargetingVBRTest() { @@ -230,6 +327,39 @@ } } + virtual void BasicRateTargetingCBR3TLTest() { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 63; + cfg_.rc_end_usage = AOM_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_usage = AOM_USAGE_REALTIME; + cfg_.g_error_resilient = 1; + + ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, + 288, 30, 1, 0, 300); + const int bitrate_array[2] = { 150, 550 }; + cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)]; + ResetModel(); + number_temporal_layers_ = 3; + target_layer_bitrate_[0] = 50 * cfg_.rc_target_bitrate / 100; + target_layer_bitrate_[1] = 70 * cfg_.rc_target_bitrate / 100; + target_layer_bitrate_[2] = cfg_.rc_target_bitrate; + framerate_factor_[0] = 4; + framerate_factor_[1] = 2; + framerate_factor_[2] = 1; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + for (int i = 0; i < number_temporal_layers_; i++) { + ASSERT_GE(effective_datarate_tl[i], target_layer_bitrate_[i] * 0.80) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_tl[i], target_layer_bitrate_[i] * 1.30) + << " The datarate for the file is greater than target by too much!"; + } + } + aom_codec_pts_t last_pts_; double timebase_; int frame_number_; // Counter for number of non-dropped/encoded frames. @@ -245,6 +375,14 @@ int denoiser_offon_test_; int denoiser_offon_period_; unsigned int aq_mode_; + int number_temporal_layers_; + // Allow for up to 3 temporal layers. + int target_layer_bitrate_[3]; + aom_svc_params_t svc_params_; + aom_svc_ref_frame_config_t ref_frame_config_; + aom_svc_layer_id_t layer_id_; + double effective_datarate_tl[3]; + int framerate_factor_[3]; }; // namespace // Check basic rate targeting for VBR mode. @@ -295,6 +433,11 @@ ChangingDropFrameThreshTest(); } +// Check basic rate targeting for CBR, for 3 temporal layers. +TEST_P(DatarateTestRealtime, BasicRateTargetingCBR3TL) { + BasicRateTargetingCBR3TLTest(); +} + AV1_INSTANTIATE_TEST_CASE(DatarateTestLarge, ::testing::Values(::libaom_test::kOnePassGood, ::libaom_test::kRealTime),
diff --git a/test/encode_test_driver.h b/test/encode_test_driver.h index 4f3f855..65f8944 100644 --- a/test/encode_test_driver.h +++ b/test/encode_test_driver.h
@@ -119,6 +119,21 @@ ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); } + void Control(int ctrl_id, struct aom_svc_layer_id *arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + + void Control(int ctrl_id, struct aom_svc_ref_frame_config *arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + + void Control(int ctrl_id, struct aom_svc_params *arg) { + const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg); + ASSERT_EQ(AOM_CODEC_OK, res) << EncoderError(); + } + #if CONFIG_AV1_ENCODER void Control(int ctrl_id, aom_active_map_t *arg) { const aom_codec_err_t res = aom_codec_control_(&encoder_, ctrl_id, arg);