rtc: Add simulcast mode to svc encoder.c
Example is added for mode 11 in the svc
sample encoder: svc_encoder_rtc
Simulcast example is added for case of
(3 spatial layers, 3 temporal layers).
No changes are needed inside the encoder,
spatial simulcast can be done via the existing
control SVC_SET_REF_FRAME_CONFIG. Only need
to make sure there is only temporal prediction within
the spatial layer, and no overlap in the reference
buffers slots across the spatial layers.
Add simulcast mode to the SVC unittests.
Update comment regarding assert check for
inter frames in nonrd_pickmode.
Change-Id: I1f7b2815386a12ebb3b3925f8fc62e4580581a40
diff --git a/aom/aomcx.h b/aom/aomcx.h
index 53bdc1e..f275890 100644
--- a/aom/aomcx.h
+++ b/aom/aomcx.h
@@ -1677,10 +1677,10 @@
/*!brief Parameters for setting ref frame config */
typedef struct aom_svc_ref_frame_config {
- // 7 references: LAST_FRAME (0), LAST2_FRAME(1), LAST3_FRAME(2),
- // GOLDEN_FRAME(3), BWDREF_FRAME(4), ALTREF2_FRAME(5), ALTREF_FRAME(6).
+ // 7 references: The index 0 - 6 refers to the references:
+ // last(0), last2(1), last3(2), golden(3), bwdref(4), altref2(5), altref(6).
int reference[7]; /**< Reference flag for each of the 7 references. */
- /*! Buffer slot index for each of 7 references. */
+ /*! Buffer slot index for each of 7 references indexed above. */
int ref_idx[7];
int refresh[8]; /**< Refresh flag for each of the 8 slots. */
} aom_svc_ref_frame_config_t;
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 9b774e2..2b46559 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1805,10 +1805,10 @@
use_ref_frame[ALTREF_FRAME] = use_alt_ref_frame;
use_ref_frame[GOLDEN_FRAME] = use_golden_ref_frame;
use_ref_frame[LAST_FRAME] = use_last_ref_frame;
- // For now keep this assert on, but we should remove it for svc mode,
- // as the user may want to generate an intra-only frame (no inter-modes).
- // Remove this assert in subsequent CL when nonrd_pickmode is tested for the
- // case of intra-only frame (no references enabled).
+ // Keep this assert on, as only 3 references are used in nonrd_pickmode
+ // (LAST, GOLDEN, ALTREF), and if all 3 are not set by user then this
+ // frame must be an intra-only frame and hence should never enter the
+ // pickmode here for inter frames.
assert(use_last_ref_frame || use_golden_ref_frame || use_alt_ref_frame);
}
diff --git a/examples/svc_encoder_rtc.cc b/examples/svc_encoder_rtc.cc
index 8bae0d1..cc23dc2 100644
--- a/examples/svc_encoder_rtc.cc
+++ b/examples/svc_encoder_rtc.cc
@@ -437,10 +437,12 @@
enc_cfg->rc_target_bitrate, enc_cfg->kf_max_dist);
}
-static int mode_to_num_temporal_layers[11] = {
- 1, 2, 3, 3, 2, 1, 1, 3, 3, 3, 3
+static int mode_to_num_temporal_layers[12] = {
+ 1, 2, 3, 3, 2, 1, 1, 3, 3, 3, 3, 3,
};
-static int mode_to_num_spatial_layers[11] = { 1, 1, 1, 1, 1, 2, 3, 2, 3, 3, 3 };
+static int mode_to_num_spatial_layers[12] = {
+ 1, 1, 1, 1, 1, 2, 3, 2, 3, 3, 3, 3,
+};
// For rate control encoding stats.
struct RateControlMetrics {
@@ -615,6 +617,7 @@
int i;
int enable_longterm_temporal_ref = 1;
int shift = (layering_mode == 8) ? 2 : 0;
+ int simulcast_mode = (layering_mode == 11);
*use_svc_control = 1;
layer_id->spatial_layer_id = spatial_layer_id;
int lag_index = 0;
@@ -1110,7 +1113,173 @@
ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 4;
}
}
- if (layer_id->spatial_layer_id > 0) {
+ break;
+ case 11:
+ // Simulcast mode for 3 spatial and 3 temporal layers.
+ // No inter-layer predicton, only prediction is temporal and single
+ // reference (LAST).
+ // No overlap in buffer slots between spatial layers. So for example,
+ // SL0 only uses slots 0 and 1.
+ // SL1 only uses slots 2 and 3.
+ // SL2 only uses slots 4 and 5.
+ // All 7 references for each inter-frame must only access buffer slots
+ // for that spatial layer.
+ // On key (super)frames: SL1 and SL2 must have no references set
+ // and must refresh all the slots for that layer only (so 2 and 3
+ // for SL1, 4 and 5 for SL2). The base SL0 will be labelled internally
+ // as a Key frame (refresh all slots). SL1/SL2 will be labelled
+ // internally as Intra-only frames that allow that stream to be decoded.
+ // These conditions will allow for each spatial stream to be
+ // independently decodeable.
+
+ // Initialize all references to 0 (don't use reference).
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->reference[i] = 0;
+ // Initialize as no refresh/update for all slots.
+ for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 0;
+
+ if (is_key_frame) {
+ if (layer_id->spatial_layer_id == 0) {
+ // Assign LAST/GOLDEN to slot 0/1.
+ // Refesh slots 0 and 1 for SL0.
+ // SL0: this will get set to KEY frame internally.
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 1;
+ ref_frame_config->refresh[0] = 1;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Assign LAST/GOLDEN to slot 2/3.
+ // Refesh slots 2 and 3 for SL1.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
+ ref_frame_config->refresh[2] = 1;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Assign LAST/GOLDEN to slot 4/5.
+ // Refresh slots 4 and 5 for SL2.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 5;
+ ref_frame_config->refresh[4] = 1;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if (superframe_cnt % 4 == 0) {
+ // Base temporal layer: TL0
+ layer_id->temporal_layer_id = 0;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST. Assign all references to either slot
+ // 0 or 1. Here we assign LAST to slot 0, all others to 1.
+ // Update slot 0 (LAST).
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ ref_frame_config->refresh[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST. Assign all references to either slot
+ // 2 or 3. Here we assign LAST to slot 2, all others to 3.
+ // Update slot 2 (LAST).
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ ref_frame_config->refresh[2] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST. Assign all references to either slot
+ // 4 or 5. Here we assign LAST to slot 4, all others to 5.
+ // Update slot 4 (LAST).
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ ref_frame_config->refresh[4] = 1;
+ }
+ } else if ((superframe_cnt - 1) % 4 == 0) {
+ // First top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0). Assign other references to slot 1.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2). Assign other references to slot 3.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ }
+ } else if ((superframe_cnt - 2) % 4 == 0) {
+ // Middle temporal enhancement layer: TL1
+ layer_id->temporal_layer_id = 1;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0).
+ // Set GOLDEN to slot 1 and update slot 1.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 0;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2).
+ // Set GOLDEN to slot 3 and update slot 3.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 2;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4).
+ // Set GOLDEN to slot 5 and update slot 5.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 4;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if ((superframe_cnt - 3) % 4 == 0) {
+ // Second top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 1). Assign other references to slot 0.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 3). Assign other references to slot 2.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 2;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 3;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 5). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[SVC_LAST_FRAME] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[SVC_LAST_FRAME] = 5;
+ }
+ }
+ if (!simulcast_mode && layer_id->spatial_layer_id > 0) {
// Always reference GOLDEN (inter-layer prediction).
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
if (ksvc_mode) {
@@ -1128,8 +1297,8 @@
// allow for top spatial layer to use additional temporal reference.
// Additional reference is only updated on base temporal layer, every
// 10 TL0 frames here.
- if (enable_longterm_temporal_ref && layer_id->spatial_layer_id == 2 &&
- layering_mode == 8) {
+ if (!simulcast_mode && enable_longterm_temporal_ref &&
+ layer_id->spatial_layer_id == 2 && layering_mode == 8) {
ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = REF_FRAMES - 1;
if (!is_key_frame) ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
@@ -1710,10 +1879,14 @@
frame_cnt_layer[layer] += 1;
got_data = 0;
+ // For simulcast (mode 11): write out each spatial layer to the file.
+ int ss_layers_write = (app_input.layering_mode == 11)
+ ? layer_id.spatial_layer_id + 1
+ : ss_number_layers;
while ((pkt = aom_codec_get_cx_data(&codec, &iter))) {
switch (pkt->kind) {
case AOM_CODEC_CX_FRAME_PKT:
- for (int sl = layer_id.spatial_layer_id; sl < ss_number_layers;
+ for (int sl = layer_id.spatial_layer_id; sl < ss_layers_write;
++sl) {
for (int tl = layer_id.temporal_layer_id; tl < ts_number_layers;
++tl) {
diff --git a/test/svc_datarate_test.cc b/test/svc_datarate_test.cc
index d99d6a3..04f2fd7 100644
--- a/test/svc_datarate_test.cc
+++ b/test/svc_datarate_test.cc
@@ -94,6 +94,7 @@
rps_recovery_frame_ = 0;
user_define_frame_qp_ = 0;
set_speed_per_layer_ = false;
+ simulcast_mode_ = false;
}
virtual void PreEncodeFrameHook(::libaom_test::VideoSource *video,
@@ -144,7 +145,7 @@
video->frame(), &layer_id_, &ref_frame_config_, &ref_frame_comp_pred_,
spatial_layer_id, multi_ref_, comp_pred_,
(video->frame() % cfg_.kf_max_dist) == 0, dynamic_enable_disable_mode_,
- rps_mode_, rps_recovery_frame_);
+ rps_mode_, rps_recovery_frame_, simulcast_mode_);
if (intra_only_ == 1 && frame_sync_ > 0) {
// Set an Intra-only frame on SL0 at frame_sync_.
// In order to allow decoding to start on SL0 in mid-sequence we need to
@@ -254,6 +255,15 @@
last_pts_ = pkt->data.frame.pts;
superframe_cnt_++;
}
+ // For simulcast mode: verify that for first frame to start decoding,
+ // for SL > 0, are Intra-only frames (not Key), whereas SL0 is Key.
+ if (simulcast_mode_ && superframe_cnt_ == (int)frame_to_start_decoding_) {
+ if (layer_id_.spatial_layer_id > 0) {
+ EXPECT_NE(pkt->data.frame.flags & AOM_FRAME_IS_KEY, AOM_FRAME_IS_KEY);
+ } else if (layer_id_.spatial_layer_id == 0) {
+ EXPECT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_KEY, AOM_FRAME_IS_KEY);
+ }
+ }
}
virtual void EndPassHook() {
@@ -279,6 +289,11 @@
if (layer_to_decode_ == 0 && frame_sync_ > 0 &&
(layer_frame_cnt_ - 1) % 3 != 0)
return 0;
+ } else if (simulcast_mode_) {
+ // Only start decoding at frames_to_start_decoding_ and only
+ // for top spatial layer SL2 (layer_to_decode_).
+ if (current_video_frame_ < frame_to_start_decoding_) return 0;
+ if (layer_id_.spatial_layer_id < (int)layer_to_decode_) return 0;
}
return 1;
}
@@ -345,13 +360,301 @@
}
}
+ // Simulcast mode for 3 spatial and 3 temporal layers.
+ // No inter-layer predicton, only prediction is temporal and single
+ // reference (LAST).
+ // No overlap in buffer slots between spatial layers. So for example,
+ // SL0 only uses slots 0 and 1.
+ // SL1 only uses slots 2 and 3.
+ // SL2 only uses slots 4 and 5.
+ // All 7 references for each inter-frame must only access buffer slots
+ // for that spatial layer.
+ // On key (super)frames: SL1 and SL2 must have no references set
+ // and must refresh all the slots for that layer only (so 2 and 3
+ // for SL1, 4 and 5 for SL2). The base SL0 will be labelled internally
+ // as a Key frame (refresh all slots). SL1/SL2 will be labelled
+ // internally as Intra-only frames that allow that stream to be decoded.
+ // These conditions will allow for each spatial stream to be
+ // independently decodeable.
+ static void ref_config_simulcast3SL3TL(
+ aom_svc_ref_frame_config_t *ref_frame_config,
+ aom_svc_layer_id_t *layer_id, int is_key_frame, int superframe_cnt) {
+ int i;
+ // Initialize all references to 0 (don't use reference).
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->reference[i] = 0;
+ // Initialize as no refresh/update for all slots.
+ for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++) ref_frame_config->ref_idx[i] = 0;
+
+ if (is_key_frame) {
+ if (layer_id->spatial_layer_id == 0) {
+ // Assign LAST/GOLDEN to slot 0/1.
+ // Refesh slots 0 and 1 for SL0.
+ // SL0: this will get set to KEY frame internally.
+ ref_frame_config->ref_idx[0] = 0;
+ ref_frame_config->ref_idx[3] = 1;
+ ref_frame_config->refresh[0] = 1;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Assign LAST/GOLDEN to slot 2/3.
+ // Refesh slots 2 and 3 for SL1.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[2] = 1;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Assign LAST/GOLDEN to slot 4/5.
+ // Refresh slots 4 and 5 for SL2.
+ // This will get set to Intra-only frame internally.
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->ref_idx[3] = 5;
+ ref_frame_config->refresh[4] = 1;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if (superframe_cnt % 4 == 0) {
+ // Base temporal layer: TL0
+ layer_id->temporal_layer_id = 0;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST. Assign all references to either slot
+ // 0 or 1. Here we assign LAST to slot 0, all others to 1.
+ // Update slot 0 (LAST).
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 0;
+ ref_frame_config->refresh[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST. Assign all references to either slot
+ // 2 or 3. Here we assign LAST to slot 2, all others to 3.
+ // Update slot 2 (LAST).
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->refresh[2] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST. Assign all references to either slot
+ // 4 or 5. Here we assign LAST to slot 4, all others to 5.
+ // Update slot 4 (LAST).
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->refresh[4] = 1;
+ }
+ } else if ((superframe_cnt - 1) % 4 == 0) {
+ // First top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0). Assign other references to slot 1.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 0;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2). Assign other references to slot 3.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 2;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[0] = 4;
+ }
+ } else if ((superframe_cnt - 2) % 4 == 0) {
+ // Middle temporal enhancement layer: TL1
+ layer_id->temporal_layer_id = 1;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 0).
+ // Set GOLDEN to slot 1 and update slot 1.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 0;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 2).
+ // Set GOLDEN to slot 3 and update slot 3.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 4).
+ // Set GOLDEN to slot 5 and update slot 5.
+ // This will be used as reference for next TL2.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 5;
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if ((superframe_cnt - 3) % 4 == 0) {
+ // Second top temporal enhancement layer: TL2
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) { // SL0
+ // Reference LAST (slot 1). Assign other references to slot 0.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) { // SL1
+ // Reference LAST (slot 3). Assign other references to slot 2.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 2;
+ ref_frame_config->ref_idx[0] = 3;
+ } else if (layer_id->spatial_layer_id == 2) { // SL2
+ // Reference LAST (slot 5). Assign other references to slot 4.
+ // No update/refresh on any slots.
+ ref_frame_config->reference[0] = 1;
+ for (i = 0; i < INTER_REFS_PER_FRAME; i++)
+ ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[0] = 5;
+ }
+ }
+ }
+
+ // 3 spatial and 3 temporal layer.
+ // Overlap in the buffer slot updates: the slots 3 and 4 updated by
+ // first TL2 are reused for update in TL1 superframe.
+ static void ref_config_3SL3TL(aom_svc_ref_frame_config_t *ref_frame_config,
+ aom_svc_layer_id_t *layer_id, int is_key_frame,
+ int superframe_cnt) {
+ if (superframe_cnt % 4 == 0) {
+ // Base temporal layer.
+ layer_id->temporal_layer_id = 0;
+ if (layer_id->spatial_layer_id == 0) {
+ // Reference LAST, update LAST.
+ // Set all buffer_idx to 0.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->refresh[0] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
+ // GOLDEN (and all other refs) to slot 0.
+ // Update slot 1 (LAST).
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 1;
+ ref_frame_config->refresh[1] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
+ // GOLDEN (and all other refs) to slot 1.
+ // Update slot 2 (LAST).
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 1;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->refresh[2] = 1;
+ }
+ } else if ((superframe_cnt - 1) % 4 == 0) {
+ // First top temporal enhancement layer.
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) {
+ // Reference LAST (slot 0).
+ // Set GOLDEN to slot 3 and update slot 3.
+ // Set all other buffer_idx to slot 0.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
+ // GOLDEN (and all other refs) to slot 3.
+ // Set LAST2 to slot 4 and Update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 1;
+ ref_frame_config->ref_idx[1] = 4;
+ ref_frame_config->refresh[4] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
+ // GOLDEN (and all other refs) to slot 4.
+ // No update.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[0] = 2;
+ }
+ } else if ((superframe_cnt - 2) % 4 == 0) {
+ // Middle temporal enhancement layer.
+ layer_id->temporal_layer_id = 1;
+ if (layer_id->spatial_layer_id == 0) {
+ // Reference LAST.
+ // Set all buffer_idx to 0.
+ // Set GOLDEN to slot 3 and update slot 3.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
+ // GOLDEN (and all other refs) to slot 3.
+ // Set LAST2 to slot 4 and update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
+ ref_frame_config->ref_idx[0] = 1;
+ ref_frame_config->ref_idx[2] = 4;
+ ref_frame_config->refresh[4] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
+ // GOLDEN (and all other refs) to slot 4.
+ // Set LAST2 to slot 5 and update slot 5.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
+ ref_frame_config->ref_idx[0] = 2;
+ ref_frame_config->ref_idx[2] = 5;
+ ref_frame_config->refresh[5] = 1;
+ }
+ } else if ((superframe_cnt - 3) % 4 == 0) {
+ // Second top temporal enhancement layer.
+ layer_id->temporal_layer_id = 2;
+ if (layer_id->spatial_layer_id == 0) {
+ // Set LAST to slot 3 and reference LAST.
+ // Set GOLDEN to slot 3 and update slot 3.
+ // Set all other buffer_idx to 0.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 3;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
+ } else if (layer_id->spatial_layer_id == 1) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 4,
+ // GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 4;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->ref_idx[1] = 4;
+ ref_frame_config->refresh[4] = 1;
+ } else if (layer_id->spatial_layer_id == 2) {
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 5,
+ // GOLDEN to slot 4. No update.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
+ ref_frame_config->ref_idx[0] = 5;
+ ref_frame_config->ref_idx[3] = 4;
+ }
+ }
+ if (layer_id->spatial_layer_id > 0) {
+ // Always reference GOLDEN (inter-layer prediction).
+ ref_frame_config->reference[3] = 1;
+ if (is_key_frame && layer_id->spatial_layer_id > 0) {
+ // On superframes whose base is key: remove LAST since GOLDEN
+ // is used as reference.
+ ref_frame_config->reference[0] = 0;
+ }
+ }
+ }
+
// Layer pattern configuration.
virtual int set_layer_pattern(
int frame_cnt, aom_svc_layer_id_t *layer_id,
aom_svc_ref_frame_config_t *ref_frame_config,
aom_svc_ref_frame_comp_pred_t *ref_frame_comp_pred, int spatial_layer,
int multi_ref, int comp_pred, int is_key_frame,
- int dynamic_enable_disable_mode, int rps_mode, int rps_recovery_frame) {
+ int dynamic_enable_disable_mode, int rps_mode, int rps_recovery_frame,
+ int simulcast_mode) {
int lag_index = 0;
int base_count = frame_cnt >> 2;
layer_id->spatial_layer_id = spatial_layer;
@@ -506,128 +809,21 @@
// Reference GOLDEN.
if (layer_id->spatial_layer_id > 0) ref_frame_config->reference[3] = 1;
} else if (number_temporal_layers_ == 3 && number_spatial_layers_ == 3) {
- // 3 spatial and 3 temporal layer.
- // Overlap in the buffer slot updates: the slots 3 and 4 updated by
- // first TL2 are reused for update in TL1 superframe.
- if (superframe_cnt_ % 4 == 0) {
- // Base temporal layer.
- layer_id->temporal_layer_id = 0;
- if (layer_id->spatial_layer_id == 0) {
- // Reference LAST, update LAST.
- // Set all buffer_idx to 0.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->refresh[0] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 0.
- // Update slot 1 (LAST).
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->refresh[1] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 1.
- // Update slot 2 (LAST).
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 1;
- ref_frame_config->ref_idx[0] = 2;
- ref_frame_config->refresh[2] = 1;
+ if (simulcast_mode) {
+ ref_config_simulcast3SL3TL(ref_frame_config, layer_id, is_key_frame,
+ superframe_cnt_);
+ } else {
+ ref_config_3SL3TL(ref_frame_config, layer_id, is_key_frame,
+ superframe_cnt_);
+ // Allow for top spatial layer to use additional temporal reference.
+ // Additional reference is only updated on base temporal layer, every
+ // 10 TL0 frames here.
+ if (multi_ref && layer_id->spatial_layer_id == 2) {
+ ref_frame_config->ref_idx[6] = 7;
+ if (!is_key_frame) ref_frame_config->reference[6] = 1;
+ if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
+ ref_frame_config->refresh[7] = 1;
}
- } else if ((superframe_cnt_ - 1) % 4 == 0) {
- // First top temporal enhancement layer.
- layer_id->temporal_layer_id = 2;
- if (layer_id->spatial_layer_id == 0) {
- // Reference LAST (slot 0).
- // Set GOLDEN to slot 3 and update slot 3.
- // Set all other buffer_idx to slot 0.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->refresh[3] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 3.
- // Set LAST2 to slot 4 and Update slot 4.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
- ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->ref_idx[1] = 4;
- ref_frame_config->refresh[4] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 4.
- // No update.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
- ref_frame_config->ref_idx[0] = 2;
- }
- } else if ((superframe_cnt_ - 2) % 4 == 0) {
- // Middle temporal enhancement layer.
- layer_id->temporal_layer_id = 1;
- if (layer_id->spatial_layer_id == 0) {
- // Reference LAST.
- // Set all buffer_idx to 0.
- // Set GOLDEN to slot 3 and update slot 3.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->refresh[3] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 3.
- // Set LAST2 to slot 4 and update slot 4.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
- ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->ref_idx[2] = 4;
- ref_frame_config->refresh[4] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 4.
- // Set LAST2 to slot 5 and update slot 5.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
- ref_frame_config->ref_idx[0] = 2;
- ref_frame_config->ref_idx[2] = 5;
- ref_frame_config->refresh[5] = 1;
- }
- } else if ((superframe_cnt_ - 3) % 4 == 0) {
- // Second top temporal enhancement layer.
- layer_id->temporal_layer_id = 2;
- if (layer_id->spatial_layer_id == 0) {
- // Set LAST to slot 3 and reference LAST.
- // Set GOLDEN to slot 3 and update slot 3.
- // Set all other buffer_idx to 0.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 3;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->refresh[3] = 1;
- } else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 4,
- // GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 4;
- ref_frame_config->ref_idx[3] = 3;
- ref_frame_config->ref_idx[1] = 4;
- ref_frame_config->refresh[4] = 1;
- } else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 5,
- // GOLDEN to slot 4. No update.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 5;
- ref_frame_config->ref_idx[3] = 4;
- }
- }
- if (layer_id->spatial_layer_id > 0) {
- // Always reference GOLDEN (inter-layer prediction).
- ref_frame_config->reference[3] = 1;
- if (is_key_frame && layer_id->spatial_layer_id > 0) {
- // On superframes whose base is key: remove LAST since GOLDEN
- // is used as reference.
- ref_frame_config->reference[0] = 0;
- }
- }
- // Allow for top spatial layer to use additional temporal reference.
- // Additional reference is only updated on base temporal layer, every
- // 10 TL0 frames here.
- if (multi_ref && layer_id->spatial_layer_id == 2) {
- ref_frame_config->ref_idx[6] = 7;
- if (!is_key_frame) ref_frame_config->reference[6] = 1;
- if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
- ref_frame_config->refresh[7] = 1;
}
}
// If the top spatial layer is first-time encoded in mid-sequence
@@ -1090,6 +1286,60 @@
EXPECT_EQ((int)GetMismatchFrames(), 150);
}
+ virtual void BasicRateTargetingSVC3TL3SLSimulcast() {
+ cfg_.rc_buf_initial_sz = 500;
+ cfg_.rc_buf_optimal_sz = 500;
+ cfg_.rc_buf_sz = 1000;
+ cfg_.rc_dropframe_thresh = 0;
+ cfg_.rc_min_quantizer = 0;
+ cfg_.rc_max_quantizer = 56;
+ cfg_.rc_end_usage = AOM_CBR;
+ cfg_.g_lag_in_frames = 0;
+ cfg_.g_error_resilient = 0;
+ cfg_.kf_max_dist = 150;
+ cfg_.kf_min_dist = 150;
+ int num_frames = 300;
+ ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
+ 288, 30, 1, 0, num_frames);
+ const int bitrate_array[2] = { 500, 1000 };
+ cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)];
+ ResetModel();
+ simulcast_mode_ = 1;
+ frame_to_start_decoding_ = cfg_.kf_max_dist;
+ layer_to_decode_ = 2; // SL2
+ number_temporal_layers_ = 3;
+ number_spatial_layers_ = 3;
+ // SL0
+ const int bitrate_sl0 = 1 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[0] = 50 * bitrate_sl0 / 100;
+ target_layer_bitrate_[1] = 70 * bitrate_sl0 / 100;
+ target_layer_bitrate_[2] = bitrate_sl0;
+ // SL1
+ const int bitrate_sl1 = 3 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[3] = 50 * bitrate_sl1 / 100;
+ target_layer_bitrate_[4] = 70 * bitrate_sl1 / 100;
+ target_layer_bitrate_[5] = bitrate_sl1;
+ // SL2
+ const int bitrate_sl2 = 4 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[6] = 50 * bitrate_sl2 / 100;
+ target_layer_bitrate_[7] = 70 * bitrate_sl2 / 100;
+ target_layer_bitrate_[8] = bitrate_sl2;
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ // Only SL2 layer is decoded.
+ for (int tl = 0; tl < number_temporal_layers_; tl++) {
+ int i = layer_to_decode_ * number_temporal_layers_ + tl;
+ ASSERT_GE(effective_datarate_tl[i], target_layer_bitrate_[i] * 0.6)
+ << " The datarate for the file is lower than target by too much!";
+ ASSERT_LE(effective_datarate_tl[i], target_layer_bitrate_[i] * 1.7)
+ << " The datarate for the file is greater than target by too much!";
+ }
+ // Only top spatial layer (SL2) is decoded, starting at frame 150
+ // (frame_to_start_decoding_), so there (300 - 150) / 2 = 75
+ // non-reference frames, so mismatch is 75.
+ int num_mismatch = (num_frames - frame_to_start_decoding_) / 2;
+ EXPECT_EQ((int)GetMismatchFrames(), num_mismatch);
+ }
+
virtual void BasicRateTargetingSVC1TL2SLIntraOnlyTest() {
cfg_.rc_buf_initial_sz = 500;
cfg_.rc_buf_optimal_sz = 500;
@@ -2133,6 +2383,7 @@
int screen_mode_;
int rps_mode_;
int rps_recovery_frame_;
+ int simulcast_mode_;
int user_define_frame_qp_;
int frame_qp_;
@@ -2199,6 +2450,14 @@
BasicRateTargetingSVC3TL3SLIntraMidSeqDecodeAll();
}
+// Check simulcast mode for 3 spatial layers, 3 temporal,
+// Key frame is inserted on base SLO in mid-stream, and verify that the
+// top spatial layer (SL2) case be decoded, starting with an Intra-only frame.
+// Verify that we can decode all frames for SL2 with no mismatch.
+TEST_P(DatarateTestSVC, BasicRateTargetingSVC3TL3SLSimulcast) {
+ BasicRateTargetingSVC3TL3SLSimulcast();
+}
+
// Check basic rate targeting for CBR, for 2 spatial layers, 1 temporal,
// with Intra-only frame inserted in the stream.
TEST_P(DatarateTestSVC, BasicRateTargetingSVC1TL2SLIntraOnly) {
