rtc-svc: Allow for fixed SVC modes Allow for fixed SVC modes: this is for when the user sets the number of spatial and temporal layers but does not use the SET_SVC_REF_FRAME_CONFIG control (flexible mode) or the "flags" (AOM_EFLAG_) in the encode call. In such a case the pattern is set to a fixed structure, and is only allowed for up to 3 spatial or temporal layers. Also add ksvc_fixed_mode flag so fixed SVC supports full SVC and KVC. The (renamed) flag svc->set_ref_frame_config indicates the SVC is set using the svc->(refresh/ref_idx/reference), in either flexible or fixed mode. For the sample enocder, svc_encoder_rtc.c, fixed mode is tested when layering_mode = -1. Also fix the (SL=2,TL=3) pattern in the sample encoder to have no updates on top spatial top temporal layers. Change-Id: I84ea2c8edac393e8aa0390f1d7fdb540c183fc58
diff --git a/aom/aomcx.h b/aom/aomcx.h index b4eeeb1..8b9a398 100644 --- a/aom/aomcx.h +++ b/aom/aomcx.h
@@ -1449,6 +1449,8 @@ int layer_target_bitrate[AOM_MAX_LAYERS]; /*! Frame rate factor for each temporal layer */ int framerate_factor[AOM_MAX_TS_LAYERS]; + /*! Flag to do KSVC for fixed mode */ + int ksvc_fixed_mode; } aom_svc_params_t; /*!brief Parameters for setting ref frame config */
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index a2acc61..bb4e7f8 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -2236,6 +2236,9 @@ cpi_lap->common.error.setjmp = 1; } + if (cpi->use_svc && cpi->svc.use_flexible_mode == 0 && flags == 0) + av1_set_svc_fixed_mode(cpi); + // Note(yunqing): While applying encoding flags, always start from enabling // all, and then modifying according to the flags. Previous frame's flags are // overwritten. @@ -2690,6 +2693,7 @@ cm->number_temporal_layers = params->number_temporal_layers; cpi->svc.number_spatial_layers = params->number_spatial_layers; cpi->svc.number_temporal_layers = params->number_temporal_layers; + cpi->svc.ksvc_fixed_mode = params->ksvc_fixed_mode; if (cm->number_spatial_layers > 1 || cm->number_temporal_layers > 1) { unsigned int sl, tl; cpi->use_svc = 1; @@ -2725,13 +2729,15 @@ AV1_COMP *const cpi = ctx->ppi->cpi; aom_svc_ref_frame_config_t *const data = va_arg(args, aom_svc_ref_frame_config_t *); - cpi->svc.external_ref_frame_config = 1; + cpi->svc.set_ref_frame_config = 1; for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; ++i) { cpi->svc.reference[i] = data->reference[i]; cpi->svc.ref_idx[i] = data->ref_idx[i]; } for (unsigned int i = 0; i < REF_FRAMES; ++i) cpi->svc.refresh[i] = data->refresh[i]; + cpi->svc.use_flexible_mode = 1; + cpi->svc.ksvc_fixed_mode = 0; return AOM_CODEC_OK; }
diff --git a/av1/encoder/av1_temporal_denoiser.c b/av1/encoder/av1_temporal_denoiser.c index 6c5bb93..730711e 100644 --- a/av1/encoder/av1_temporal_denoiser.c +++ b/av1/encoder/av1_temporal_denoiser.c
@@ -415,7 +415,7 @@ return; } - if (svc->external_ref_frame_config) { + if (svc->set_ref_frame_config) { int i; for (i = 0; i < REF_FRAMES; i++) { if (svc->refresh[svc->spatial_layer_id] & (1 << i)) @@ -501,7 +501,7 @@ int refresh_alt, int refresh_gld, int refresh_lst, int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) { int fail = 0; - if (svc->external_ref_frame_config) { + if (svc->set_ref_frame_config) { int i; for (i = 0; i < REF_FRAMES; i++) { if (cm->current_frame.frame_type == KEY_FRAME ||
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index da7ec44..a4ee6d5 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -141,7 +141,7 @@ const ExtRefreshFrameFlagsInfo *const ext_refresh_frame_flags) { // Droppable frame is only used by external refresh flags. VoD setting won't // trigger its use case. - if (svc->external_ref_frame_config) + if (svc->set_ref_frame_config) return svc->non_reference_frame; else if (ext_refresh_frame_flags->update_pending) return !(ext_refresh_frame_flags->alt_ref_frame || @@ -733,7 +733,7 @@ int refresh_mask = 0; if (ext_refresh_frame_flags->update_pending) { - if (svc->external_ref_frame_config) { + if (svc->set_ref_frame_config) { for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; i++) { int ref_frame_map_idx = svc->ref_idx[i]; refresh_mask |= svc->refresh[ref_frame_map_idx] << ref_frame_map_idx; @@ -1304,7 +1304,7 @@ if (!ext_flags->refresh_frame.update_pending) { av1_get_ref_frames(cpi, &cpi->ref_buffer_stack); - } else if (cpi->svc.external_ref_frame_config) { + } else if (cpi->svc.set_ref_frame_config) { for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; i++) cm->remapped_ref_idx[i] = cpi->svc.ref_idx[i]; }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index c37ed29..9407082 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -560,7 +560,7 @@ // Set init SVC parameters. cpi->use_svc = 0; - cpi->svc.external_ref_frame_config = 0; + cpi->svc.set_ref_frame_config = 0; cpi->svc.non_reference_frame = 0; cpi->svc.number_spatial_layers = 1; cpi->svc.number_temporal_layers = 1; @@ -780,7 +780,7 @@ set_tile_info(cm, &cpi->oxcf.tile_cfg); - if (!cpi->svc.external_ref_frame_config) + if (!cpi->svc.set_ref_frame_config) cpi->ext_flags.refresh_frame.update_pending = 0; cpi->ext_flags.refresh_frame_context_pending = 0; @@ -3922,7 +3922,7 @@ return AOM_CODEC_OK; } -static void svc_set_updates_external_ref_frame_config( +static void svc_set_updates_ref_frame_config( ExtRefreshFrameFlagsInfo *const ext_refresh_frame_flags, SVC *const svc) { ext_refresh_frame_flags->update_pending = 1; ext_refresh_frame_flags->last_frame = svc->refresh[svc->ref_idx[0]]; @@ -3983,7 +3983,7 @@ av1_use_as_reference(&ext_flags->ref_frame_flags, ref); } else { - if (cpi->svc.external_ref_frame_config) { + if (cpi->svc.set_ref_frame_config) { int ref = svc_set_references_external_ref_frame_config(cpi); av1_use_as_reference(&ext_flags->ref_frame_flags, ref); } @@ -4011,9 +4011,8 @@ ext_refresh_frame_flags->alt2_ref_frame = (upd & AOM_ALT2_FLAG) != 0; ext_refresh_frame_flags->update_pending = 1; } else { - if (cpi->svc.external_ref_frame_config) - svc_set_updates_external_ref_frame_config(ext_refresh_frame_flags, - &cpi->svc); + if (cpi->svc.set_ref_frame_config) + svc_set_updates_ref_frame_config(ext_refresh_frame_flags, &cpi->svc); else ext_refresh_frame_flags->update_pending = 0; }
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index 33befa6..d1f0893 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c
@@ -2177,7 +2177,7 @@ int gld_idx = 0; int alt_ref_idx = 0; ext_refresh_frame_flags->update_pending = 1; - svc->external_ref_frame_config = 1; + svc->set_ref_frame_config = 1; ext_flags->ref_frame_flags = 0; ext_refresh_frame_flags->last_frame = 1; ext_refresh_frame_flags->golden_frame = 0;
diff --git a/av1/encoder/svc_layercontext.c b/av1/encoder/svc_layercontext.c index 1710920..f76b34c 100644 --- a/av1/encoder/svc_layercontext.c +++ b/av1/encoder/svc_layercontext.c
@@ -30,6 +30,7 @@ svc->current_superframe = 0; svc->force_zero_mode_spatial_ref = 1; svc->num_encoded_top_layer = 0; + svc->use_flexible_mode = 0; for (int sl = 0; sl < svc->number_spatial_layers; ++sl) { for (int tl = 0; tl < svc->number_temporal_layers; ++tl) { @@ -198,7 +199,7 @@ // This is to skip searching mv for that reference if it was last // refreshed (i.e., buffer slot holding that reference was refreshed) on the // previous spatial layer(s) at the same time (current_superframe). - if (svc->external_ref_frame_config && svc->force_zero_mode_spatial_ref) { + if (svc->set_ref_frame_config && svc->force_zero_mode_spatial_ref) { int ref_frame_idx = svc->ref_idx[LAST_FRAME - 1]; if (svc->buffer_time_index[ref_frame_idx] == svc->current_superframe && svc->buffer_spatial_layer[ref_frame_idx] <= svc->spatial_layer_id - 1) @@ -243,7 +244,7 @@ svc->buffer_time_index[i] = svc->current_superframe; svc->buffer_spatial_layer[i] = svc->spatial_layer_id; } - } else if (cpi->svc.external_ref_frame_config) { + } else if (cpi->svc.set_ref_frame_config) { for (unsigned int i = 0; i < INTER_REFS_PER_FRAME; i++) { int ref_frame_map_idx = svc->ref_idx[i]; if (cpi->svc.refresh[ref_frame_map_idx]) { @@ -342,3 +343,147 @@ cpi->common.height = height; av1_update_frame_size(cpi); } + +enum { + SVC_LAST_FRAME = 0, + SVC_LAST2_FRAME, + SVC_LAST3_FRAME, + SVC_GOLDEN_FRAME, + SVC_BWDREF_FRAME, + SVC_ALTREF2_FRAME, + SVC_ALTREF_FRAME +}; + +// For fixed svc mode: fixed pattern is set based on the number of +// spatial and temporal layers, and the ksvc_fixed_mode. +void av1_set_svc_fixed_mode(AV1_COMP *const cpi) { + SVC *const svc = &cpi->svc; + int i; + assert(svc->use_flexible_mode == 0); + // Fixed SVC mode only supports at most 3 spatial or temporal layers. + assert(svc->number_spatial_layers >= 1 && svc->number_spatial_layers <= 3 && + svc->number_temporal_layers >= 1 && svc->number_temporal_layers <= 3); + svc->set_ref_frame_config = 1; + int superframe_cnt = svc->current_superframe; + // Set the referende map buffer idx for the 7 references: + // LAST_FRAME (0), LAST2_FRAME(1), LAST3_FRAME(2), GOLDEN_FRAME(3), + // BWDREF_FRAME(4), ALTREF2_FRAME(5), ALTREF_FRAME(6). + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = i; + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->reference[i] = 0; + for (i = 0; i < REF_FRAMES; i++) svc->refresh[i] = 0; + // Always reference LAST, and reference GOLDEN on SL > 0 for non-ksvc. + svc->reference[SVC_LAST_FRAME] = 1; + if (svc->spatial_layer_id > 0 && + (!svc->ksvc_fixed_mode || + cpi->common.current_frame.frame_type == KEY_FRAME)) + svc->reference[SVC_GOLDEN_FRAME] = 1; + if (svc->temporal_layer_id == 0) { + // Base temporal layer. + if (svc->spatial_layer_id == 0) { + // Set all buffer_idx to 0. Update slot 0 (LAST). + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + svc->refresh[0] = 1; + } else if (svc->spatial_layer_id == 1) { + // Set buffer_idx for LAST to slot 1, GOLDEN (and all other refs) to + // slot 0. Update slot 1 (LAST). + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + svc->ref_idx[SVC_LAST_FRAME] = 1; + svc->refresh[1] = 1; + } else if (svc->spatial_layer_id == 2) { + // Set buffer_idx for LAST to slot 2, GOLDEN (and all other refs) to + // slot 1. Update slot 2 (LAST). + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 1; + svc->ref_idx[SVC_LAST_FRAME] = 2; + svc->refresh[2] = 1; + } + } else if (svc->temporal_layer_id == 2 && (superframe_cnt - 1) % 4 == 0) { + // First top temporal enhancement layer. + if (svc->spatial_layer_id == 0) { + // Reference LAST (slot 0). + // Set GOLDEN to slot 3 and update slot 3. + // Set all other buffer_idx to slot 0. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + if (svc->spatial_layer_id < svc->number_spatial_layers - 1) { + svc->ref_idx[SVC_GOLDEN_FRAME] = 3; + svc->refresh[3] = 1; + } + } else if (svc->spatial_layer_id == 1) { + // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1, + // GOLDEN (and all other refs) to slot 3. + // Set LAST2 to slot 4 and Update slot 4. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 3; + svc->ref_idx[SVC_LAST_FRAME] = 1; + if (svc->spatial_layer_id < svc->number_spatial_layers - 1) { + svc->ref_idx[SVC_LAST2_FRAME] = 4; + svc->refresh[4] = 1; + } + } else if (svc->spatial_layer_id == 2) { + // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2, + // GOLDEN (and all other refs) to slot 4. + // No update. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 4; + svc->ref_idx[SVC_LAST_FRAME] = 2; + } + } else if (svc->temporal_layer_id == 1) { + // Middle temporal enhancement layer. + if (svc->spatial_layer_id == 0) { + // Reference LAST. + // Set all buffer_idx to 0. + // Set GOLDEN to slot 5 and update slot 5. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + if (svc->temporal_layer_id < svc->number_temporal_layers - 1) { + svc->ref_idx[SVC_GOLDEN_FRAME] = 5; + svc->refresh[5] = 1; + } + } else if (svc->spatial_layer_id == 1) { + // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1, + // GOLDEN (and all other refs) to slot 5. + // Set LAST3 to slot 6 and update slot 6. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 5; + svc->ref_idx[SVC_LAST_FRAME] = 1; + if (svc->temporal_layer_id < svc->number_temporal_layers - 1) { + svc->ref_idx[SVC_LAST3_FRAME] = 6; + svc->refresh[6] = 1; + } + } else if (svc->spatial_layer_id == 2) { + // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2, + // GOLDEN (and all other refs) to slot 6. + // Set LAST3 to slot 7 and update slot 7. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 6; + svc->ref_idx[SVC_LAST_FRAME] = 2; + if (svc->temporal_layer_id < svc->number_temporal_layers - 1) { + svc->ref_idx[SVC_LAST3_FRAME] = 7; + svc->refresh[7] = 1; + } + } + } else if (svc->temporal_layer_id == 2 && (superframe_cnt - 3) % 4 == 0) { + // Second top temporal enhancement layer. + if (svc->spatial_layer_id == 0) { + // Set LAST to slot 5 and reference LAST. + // Set GOLDEN to slot 3 and update slot 3. + // Set all other buffer_idx to 0. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + svc->ref_idx[SVC_LAST_FRAME] = 5; + if (svc->spatial_layer_id < svc->number_spatial_layers - 1) { + svc->ref_idx[SVC_GOLDEN_FRAME] = 3; + svc->refresh[3] = 1; + } + } else if (svc->spatial_layer_id == 1) { + // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 6, + // GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + svc->ref_idx[SVC_LAST_FRAME] = 6; + svc->ref_idx[SVC_GOLDEN_FRAME] = 3; + if (svc->spatial_layer_id < svc->number_spatial_layers - 1) { + svc->ref_idx[SVC_LAST2_FRAME] = 4; + svc->refresh[4] = 1; + } + } else if (svc->spatial_layer_id == 2) { + // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 7, + // GOLDEN to slot 4. No update. + for (i = 0; i < INTER_REFS_PER_FRAME; i++) svc->ref_idx[i] = 0; + svc->ref_idx[SVC_LAST_FRAME] = 7; + svc->ref_idx[SVC_GOLDEN_FRAME] = 4; + } + } +}
diff --git a/av1/encoder/svc_layercontext.h b/av1/encoder/svc_layercontext.h index 1eeba5e..44e13d6 100644 --- a/av1/encoder/svc_layercontext.h +++ b/av1/encoder/svc_layercontext.h
@@ -94,8 +94,10 @@ int temporal_layer_id; int number_spatial_layers; int number_temporal_layers; - int external_ref_frame_config; + int set_ref_frame_config; int non_reference_frame; + int use_flexible_mode; + int ksvc_fixed_mode; /*!\endcond */ /*! @@ -271,6 +273,9 @@ void av1_get_layer_resolution(const int width_org, const int height_org, const int num, const int den, int *width_out, int *height_out); + +void av1_set_svc_fixed_mode(struct AV1_COMP *const cpi); + #ifdef __cplusplus } // extern "C" #endif
diff --git a/examples/svc_encoder_rtc.c b/examples/svc_encoder_rtc.c index 24541f9..17456d8 100644 --- a/examples/svc_encoder_rtc.c +++ b/examples/svc_encoder_rtc.c
@@ -796,12 +796,10 @@ } else if (layer_id->spatial_layer_id == 1) { // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1, // GOLDEN (and all other refs) to slot 3. - // Set LAST2 to slot 4 and Update slot 4. + // No update. for (i = 0; i < INTER_REFS_PER_FRAME; i++) ref_frame_config->ref_idx[i] = 3; ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1; - ref_frame_config->ref_idx[SVC_LAST2_FRAME] = 4; - ref_frame_config->refresh[4] = 1; } } else if ((superframe_cnt - 2) % 4 == 0) { // Middle temporal enhancement layer. @@ -838,13 +836,11 @@ ref_frame_config->refresh[3] = 1; } else if (layer_id->spatial_layer_id == 1) { // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 6, - // GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4. + // GOLDEN to slot 3. No update. for (i = 0; i < INTER_REFS_PER_FRAME; i++) ref_frame_config->ref_idx[i] = 0; ref_frame_config->ref_idx[SVC_LAST_FRAME] = 6 - shift; ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3; - ref_frame_config->ref_idx[SVC_LAST2_FRAME] = 4; - ref_frame_config->refresh[4] = 1; } } if (layer_id->spatial_layer_id > 0 && !ksvc_mode) { @@ -1141,10 +1137,13 @@ unsigned int width = cfg.g_w; unsigned int height = cfg.g_h; - if (ts_number_layers != - mode_to_num_temporal_layers[app_input.layering_mode] || - ss_number_layers != mode_to_num_spatial_layers[app_input.layering_mode]) { - die("Number of layers doesn't match layering mode."); + if (app_input.layering_mode >= 0) { + if (ts_number_layers != + mode_to_num_temporal_layers[app_input.layering_mode] || + ss_number_layers != + mode_to_num_spatial_layers[app_input.layering_mode]) { + die("Number of layers doesn't match layering mode."); + } } // Y4M reader has its own allocation. @@ -1268,7 +1267,7 @@ svc_params.scaling_factor_num[1] = 1; svc_params.scaling_factor_den[1] = 2; } - + svc_params.ksvc_fixed_mode = 0; // Set to 1 to do KSVC in fixed mode. aom_codec_control(&codec, AV1E_SET_SVC_PARAMS, &svc_params); // This controls the maximum target size of the key frame. @@ -1292,15 +1291,34 @@ const aom_codec_cx_pkt_t *pkt; int layer = 0; - // Set the reference/update flags, layer_id, and reference_map - // buffer index. - set_layer_pattern(app_input.layering_mode, frame_cnt, &layer_id, - &ref_frame_config, &use_svc_control, slx, is_key_frame, - (app_input.layering_mode == 10)); - aom_codec_control(&codec, AV1E_SET_SVC_LAYER_ID, &layer_id); - if (use_svc_control) - aom_codec_control(&codec, AV1E_SET_SVC_REF_FRAME_CONFIG, - &ref_frame_config); + // For flexible mode: + if (app_input.layering_mode >= 0) { + // Set the reference/update flags, layer_id, and reference_map + // buffer index. + set_layer_pattern(app_input.layering_mode, frame_cnt, &layer_id, + &ref_frame_config, &use_svc_control, slx, + is_key_frame, (app_input.layering_mode == 10)); + aom_codec_control(&codec, AV1E_SET_SVC_LAYER_ID, &layer_id); + if (use_svc_control) + aom_codec_control(&codec, AV1E_SET_SVC_REF_FRAME_CONFIG, + &ref_frame_config); + } else { + // Only up to 3 temporal layers supported in fixed mode. + // Only need to set spatial and temporal layer_id: reference + // prediction, refresh, and buffer_idx are set internally. + layer_id.spatial_layer_id = slx; + layer_id.temporal_layer_id = 0; + if (ts_number_layers == 2) { + layer_id.temporal_layer_id = (frame_cnt % 2) != 0; + } else if (ts_number_layers == 3) { + if (frame_cnt % 2 != 0) + layer_id.temporal_layer_id = 2; + else if ((frame_cnt > 1) && ((frame_cnt - 2) % 4 == 0)) + layer_id.temporal_layer_id = 1; + } + aom_codec_control(&codec, AV1E_SET_SVC_LAYER_ID, &layer_id); + } + if (set_err_resil_frame) { // Set error_resilient per frame: off/0 for base layer and // on/1 for enhancement layer frames.