rtc-svc: Allow for second temporal reference for spatial layers
Allow for second (longer-term) temporal reference for SVC for
the top spatial layer. altref is used for this reference.
Update sample encoder, fix in nonrd_pickmode, and add unittest.
Change-Id: I2631de594b855f2219d86728fb163dc2ceccda0c
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 199cb0d..906811b 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1609,7 +1609,8 @@
const struct segmentation *const seg = &cm->seg;
const int is_small_sb = (cm->seq_params.sb_size == BLOCK_64X64);
- int use_alt_ref_frame = cpi->sf.rt_sf.use_nonrd_altref_frame;
+ // For SVC the usage of alt_ref is determined by the ref_frame_flags.
+ int use_alt_ref_frame = cpi->use_svc || cpi->sf.rt_sf.use_nonrd_altref_frame;
int use_golden_ref_frame = 1;
use_ref_frame[LAST_FRAME] = 1; // we never skip LAST
@@ -1713,9 +1714,10 @@
uint32_t spatial_var_thresh = 50;
int motion_thresh = 32;
// Adjust thresholds to make intra mode likely tested if the other
- // references (golden, alt) are skipped/not checked.
- if (cpi->sf.rt_sf.use_nonrd_altref_frame == 0 &&
- cpi->sf.rt_sf.nonrd_prune_ref_frame_search > 0) {
+ // references (golden, alt) are skipped/not checked. For now always
+ // adjust for svc mode.
+ if (cpi->use_svc || (cpi->sf.rt_sf.use_nonrd_altref_frame == 0 &&
+ cpi->sf.rt_sf.nonrd_prune_ref_frame_search > 0)) {
spatial_var_thresh = 150;
motion_thresh = 0;
}
diff --git a/examples/svc_encoder_rtc.c b/examples/svc_encoder_rtc.c
index 2b883cf..401049f 100644
--- a/examples/svc_encoder_rtc.c
+++ b/examples/svc_encoder_rtc.c
@@ -263,6 +263,7 @@
int *use_svc_control, int spatial_layer_id,
int is_key_frame, int ksvc_mode) {
int i;
+ int enable_longterm_temporal_ref = 1;
int shift = (layering_mode == 7) ? 2 : 0;
*use_svc_control = 1;
layer_id->spatial_layer_id = spatial_layer_id;
@@ -456,6 +457,14 @@
ref_frame_config->refresh[2] = 1;
ref_frame_config->reference[SVC_LAST_FRAME] = 1;
ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
+ // For 3 spatial layer case: allow for top spatial layer to use
+ // additional temporal reference. Update every 10 frames.
+ if (enable_longterm_temporal_ref) {
+ ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = REF_FRAMES - 1;
+ ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
+ if (base_count % 10 == 0)
+ ref_frame_config->refresh[REF_FRAMES - 1] = 1;
+ }
}
break;
case 7:
@@ -587,7 +596,19 @@
}
}
if (layer_id->spatial_layer_id > 0)
- ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1; // Reference GOLDEN.
+ // Reference GOLDEN.
+ ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
+ // For 3 spatial layer case 7 (where there is free buffer slot):
+ // allow for top spatial layer to use additional temporal reference.
+ // Additional reference is only updated on base temporal layer, every
+ // 10 TL0 frames here.
+ if (enable_longterm_temporal_ref && layer_id->spatial_layer_id == 2 &&
+ layering_mode == 7) {
+ ref_frame_config->ref_idx[SVC_ALTREF_FRAME] = REF_FRAMES - 1;
+ ref_frame_config->reference[SVC_ALTREF_FRAME] = 1;
+ if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
+ ref_frame_config->refresh[REF_FRAMES - 1] = 1;
+ }
break;
default: assert(0); die("Error: Unsupported temporal layering mode!\n");
}
diff --git a/test/svc_datarate_test.cc b/test/svc_datarate_test.cc
index fcd9bc5..7a8a153 100644
--- a/test/svc_datarate_test.cc
+++ b/test/svc_datarate_test.cc
@@ -265,11 +265,18 @@
for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 1;
ref_frame_config->ref_idx[0] = 2;
ref_frame_config->refresh[2] = 1;
+ if (multi_ref) {
+ ref_frame_config->ref_idx[6] = 7;
+ ref_frame_config->reference[6] = 1;
+ if (base_count % 10 == 0) ref_frame_config->refresh[7] = 1;
+ }
}
// Reference GOLDEN.
if (layer_id->spatial_layer_id > 0) ref_frame_config->reference[3] = 1;
} else if (number_temporal_layers_ == 3 && number_spatial_layers_ == 3) {
// 3 spatial and 3 temporal layer.
+ // Overlap in the buffer slot updates: the slots 3 and 4 updated by
+ // first TL2 are reused for update in TL1 superframe.
if (superframe_cnt_ % 4 == 0) {
// Base temporal layer.
layer_id->temporal_layer_id = 0;
@@ -324,56 +331,65 @@
if (layer_id->spatial_layer_id == 0) {
// Reference LAST.
// Set all buffer_idx to 0.
- // Set GOLDEN to slot 5 and update slot 5.
+ // Set GOLDEN to slot 3 and update slot 3.
for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[3] = 5;
- ref_frame_config->refresh[5] = 1;
+ ref_frame_config->ref_idx[3] = 3;
+ ref_frame_config->refresh[3] = 1;
} else if (layer_id->spatial_layer_id == 1) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 1,
- // GOLDEN (and all other refs) to slot 5.
- // Set LAST2 to slot 6 and update slot 6.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 5;
+ // GOLDEN (and all other refs) to slot 3.
+ // Set LAST2 to slot 4 and update slot 4.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 3;
ref_frame_config->ref_idx[0] = 1;
- ref_frame_config->ref_idx[2] = 6;
- ref_frame_config->refresh[6] = 1;
+ ref_frame_config->ref_idx[2] = 4;
+ ref_frame_config->refresh[4] = 1;
} else if (layer_id->spatial_layer_id == 2) {
// Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 2,
- // GOLDEN (and all other refs) to slot 6.
- // Set LAST2 to slot 6 and update slot 7.
- for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 6;
+ // GOLDEN (and all other refs) to slot 4.
+ // Set LAST2 to slot 5 and update slot 5.
+ for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 4;
ref_frame_config->ref_idx[0] = 2;
- ref_frame_config->ref_idx[2] = 7;
- ref_frame_config->refresh[7] = 1;
+ ref_frame_config->ref_idx[2] = 5;
+ ref_frame_config->refresh[5] = 1;
}
} else if ((superframe_cnt_ - 3) % 4 == 0) {
// Second top temporal enhancement layer.
layer_id->temporal_layer_id = 2;
if (layer_id->spatial_layer_id == 0) {
- // Set LAST to slot 5 and reference LAST.
+ // Set LAST to slot 3 and reference LAST.
// Set GOLDEN to slot 3 and update slot 3.
// Set all other buffer_idx to 0.
for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 5;
+ ref_frame_config->ref_idx[0] = 3;
ref_frame_config->ref_idx[3] = 3;
ref_frame_config->refresh[3] = 1;
} else if (layer_id->spatial_layer_id == 1) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 6,
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 4,
// GOLDEN to slot 3. Set LAST2 to slot 4 and update slot 4.
for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 6;
+ ref_frame_config->ref_idx[0] = 4;
ref_frame_config->ref_idx[3] = 3;
ref_frame_config->ref_idx[1] = 4;
ref_frame_config->refresh[4] = 1;
} else if (layer_id->spatial_layer_id == 2) {
- // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 7,
+ // Reference LAST and GOLDEN. Set buffer_idx for LAST to slot 5,
// GOLDEN to slot 4. No update.
for (int i = 0; i < 7; i++) ref_frame_config->ref_idx[i] = 0;
- ref_frame_config->ref_idx[0] = 7;
+ ref_frame_config->ref_idx[0] = 5;
ref_frame_config->ref_idx[3] = 4;
}
}
// Reference GOLDEN.
if (layer_id->spatial_layer_id > 0) ref_frame_config->reference[3] = 1;
+ // Allow for top spatial layer to use additional temporal reference.
+ // Additional reference is only updated on base temporal layer, every
+ // 10 TL0 frames here.
+ if (multi_ref && layer_id->spatial_layer_id == 2) {
+ ref_frame_config->ref_idx[6] = 7;
+ ref_frame_config->reference[6] = 1;
+ if (base_count % 10 == 0 && layer_id->temporal_layer_id == 0)
+ ref_frame_config->refresh[7] = 1;
+ }
}
return layer_flags;
}
@@ -458,7 +474,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
cfg_.rc_resize_mode = RESIZE_DYNAMIC;
::libaom_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30,
@@ -506,7 +522,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
288, 30, 1, 0, 300);
@@ -535,7 +551,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
288, 30, 1, 0, 300);
@@ -556,6 +572,37 @@
}
}
+ virtual void BasicRateTargetingSVC1TL3SLMultiRefTest() {
+ cfg_.rc_buf_initial_sz = 500;
+ cfg_.rc_buf_optimal_sz = 500;
+ cfg_.rc_buf_sz = 1000;
+ cfg_.rc_dropframe_thresh = 0;
+ cfg_.rc_min_quantizer = 0;
+ cfg_.rc_max_quantizer = 63;
+ cfg_.rc_end_usage = AOM_CBR;
+ cfg_.g_lag_in_frames = 0;
+ cfg_.g_error_resilient = 0;
+
+ ::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
+ 288, 30, 1, 0, 300);
+ const int bitrate_array[2] = { 500, 1000 };
+ cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)];
+ ResetModel();
+ multi_ref_ = 1;
+ number_temporal_layers_ = 1;
+ number_spatial_layers_ = 3;
+ target_layer_bitrate_[0] = 1 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[1] = 3 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[2] = 4 * cfg_.rc_target_bitrate / 8;
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ for (int i = 0; i < number_temporal_layers_ * number_spatial_layers_; i++) {
+ ASSERT_GE(effective_datarate_tl[i], target_layer_bitrate_[i] * 0.80)
+ << " The datarate for the file is lower than target by too much!";
+ ASSERT_LE(effective_datarate_tl[i], target_layer_bitrate_[i] * 1.38)
+ << " The datarate for the file is greater than target by too much!";
+ }
+ }
+
virtual void BasicRateTargetingSVC3TL3SLTest() {
cfg_.rc_buf_initial_sz = 500;
cfg_.rc_buf_optimal_sz = 500;
@@ -565,7 +612,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
::libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352,
288, 30, 1, 0, 300);
@@ -607,7 +654,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
::libaom_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60);
const int bitrate_array[2] = { 600, 1200 };
@@ -639,6 +686,48 @@
}
}
+ virtual void BasicRateTargetingSVC3TL3SLHDMultiRefTest() {
+ cfg_.rc_buf_initial_sz = 500;
+ cfg_.rc_buf_optimal_sz = 500;
+ cfg_.rc_buf_sz = 1000;
+ cfg_.rc_dropframe_thresh = 0;
+ cfg_.rc_min_quantizer = 0;
+ cfg_.rc_max_quantizer = 63;
+ cfg_.rc_end_usage = AOM_CBR;
+ cfg_.g_lag_in_frames = 0;
+ cfg_.g_error_resilient = 0;
+
+ ::libaom_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60);
+ const int bitrate_array[2] = { 600, 1200 };
+ cfg_.rc_target_bitrate = bitrate_array[GET_PARAM(4)];
+ ResetModel();
+ multi_ref_ = 1;
+ number_temporal_layers_ = 3;
+ number_spatial_layers_ = 3;
+ // SL0
+ const int bitrate_sl0 = 1 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[0] = 50 * bitrate_sl0 / 100;
+ target_layer_bitrate_[1] = 70 * bitrate_sl0 / 100;
+ target_layer_bitrate_[2] = bitrate_sl0;
+ // SL1
+ const int bitrate_sl1 = 3 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[3] = 50 * bitrate_sl1 / 100;
+ target_layer_bitrate_[4] = 70 * bitrate_sl1 / 100;
+ target_layer_bitrate_[5] = bitrate_sl1;
+ // SL2
+ const int bitrate_sl2 = 4 * cfg_.rc_target_bitrate / 8;
+ target_layer_bitrate_[6] = 50 * bitrate_sl2 / 100;
+ target_layer_bitrate_[7] = 70 * bitrate_sl2 / 100;
+ target_layer_bitrate_[8] = bitrate_sl2;
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ for (int i = 0; i < number_temporal_layers_ * number_spatial_layers_; i++) {
+ ASSERT_GE(effective_datarate_tl[i], target_layer_bitrate_[i] * 0.70)
+ << " The datarate for the file is lower than target by too much!";
+ ASSERT_LE(effective_datarate_tl[i], target_layer_bitrate_[i] * 1.4)
+ << " The datarate for the file is greater than target by too much!";
+ }
+ }
+
virtual void BasicRateTargetingSVC3TL3SLKfTest() {
cfg_.rc_buf_initial_sz = 500;
cfg_.rc_buf_optimal_sz = 500;
@@ -648,7 +737,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
cfg_.kf_mode = AOM_KF_AUTO;
cfg_.kf_min_dist = cfg_.kf_max_dist = 100;
@@ -692,7 +781,7 @@
cfg_.rc_max_quantizer = 63;
cfg_.rc_end_usage = AOM_CBR;
cfg_.g_lag_in_frames = 0;
- cfg_.g_error_resilient = 1;
+ cfg_.g_error_resilient = 0;
cfg_.g_profile = 1;
::libaom_test::Y4mVideoSource video("rush_hour_444.y4m", 0, 140);
@@ -949,6 +1038,12 @@
BasicRateTargetingSVC1TL3SLTest();
}
+// Check basic rate targeting for CBR, for 3 spatial layers, 1 temporal,
+// with additional temporal reference for top spatial layer.
+TEST_P(DatarateTestSVC, BasicRateTargetingSVC1TL3SLMultiRef) {
+ BasicRateTargetingSVC1TL3SLMultiRefTest();
+}
+
// Check basic rate targeting for CBR, for 3 spatial, 3 temporal layers.
TEST_P(DatarateTestSVC, BasicRateTargetingSVC3TL3SL) {
BasicRateTargetingSVC3TL3SLTest();
@@ -960,6 +1055,12 @@
}
// Check basic rate targeting for CBR, for 3 spatial, 3 temporal layers,
+// with additional temporal reference for top spatial layer.
+TEST_P(DatarateTestSVC, BasicRateTargetingSVC3TL3SLHDMultiRef) {
+ BasicRateTargetingSVC3TL3SLHDMultiRefTest();
+}
+
+// Check basic rate targeting for CBR, for 3 spatial, 3 temporal layers,
// for auto key frame mode with short key frame period.
TEST_P(DatarateTestSVC, BasicRateTargetingSVC3TL3SLKf) {
BasicRateTargetingSVC3TL3SLKfTest();