rtc: Add frame-dropper support to external RTC RC frame_drop_thresh and max_consec_drop are input to the frame dropper. Test coverage added for single layer. Move the reset of gf_frame_index to where the index is updated, this resolves an out of array bounds for external rc with frame droppiing. Support for the full superframe_drop mode for spatial SVC to be added. Change-Id: I08c2d7da20e1b260fd3413d10d548340cd681b3a
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index ddb8054..689af2a 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -3173,12 +3173,6 @@ ppi->num_fp_contexts = av1_compute_num_fp_contexts(ppi, &cpi->oxcf); } - // Reset gf_frame_index in case it reaches MAX_STATIC_GF_GROUP_LENGTH for - // real time encoding. - if (is_one_pass_rt_params(cpi) && - cpi->gf_frame_index == MAX_STATIC_GF_GROUP_LENGTH) - cpi->gf_frame_index = 0; - // Get the next visible frame. Invisible frames get packed with the next // visible frame. while (cpi_data.cx_data_sz >= ctx->cx_data_sz / 2 && !is_frame_visible) {
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index 7846b7f..fed15eb 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -4431,7 +4431,16 @@ static AOM_INLINE void update_gf_group_index(AV1_COMP *cpi) { // Increment the gf group index ready for the next frame. - ++cpi->gf_frame_index; + if (is_one_pass_rt_params(cpi) && + cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1) { + ++cpi->gf_frame_index; + // Reset gf_frame_index in case it reaches MAX_STATIC_GF_GROUP_LENGTH + // for real time encoding. + if (cpi->gf_frame_index == MAX_STATIC_GF_GROUP_LENGTH) + cpi->gf_frame_index = 0; + } else { + ++cpi->gf_frame_index; + } } static void update_fb_of_context_type(const AV1_COMP *const cpi,
diff --git a/av1/ratectrl_rtc.cc b/av1/ratectrl_rtc.cc index cfcea24..cd9597b 100644 --- a/av1/ratectrl_rtc.cc +++ b/av1/ratectrl_rtc.cc
@@ -39,6 +39,8 @@ undershoot_pct = overshoot_pct = 50; max_intra_bitrate_pct = 50; max_inter_bitrate_pct = 0; + frame_drop_thresh = 0; + max_consec_drop = 0; framerate = 30.0; ss_number_layers = 1; ts_number_layers = 1; @@ -124,7 +126,8 @@ oxcf->pass = AOM_RC_ONE_PASS; oxcf->q_cfg.aq_mode = rc_cfg.aq_mode ? CYCLIC_REFRESH_AQ : NO_AQ; oxcf->tune_cfg.content = AOM_CONTENT_DEFAULT; - oxcf->rc_cfg.drop_frames_water_mark = 0; + oxcf->rc_cfg.drop_frames_water_mark = rc_cfg.frame_drop_thresh; + rc->max_consec_drop = rc_cfg.max_consec_drop; oxcf->tool_cfg.bit_depth = AOM_BITS_8; oxcf->tool_cfg.superblock_size = AOM_SUPERBLOCK_SIZE_DYNAMIC; oxcf->algo_cfg.loopfilter_control = LOOPFILTER_ALL; @@ -185,6 +188,8 @@ oxcf->rc_cfg.maximum_buffer_size_ms = rc_cfg.buf_sz; oxcf->rc_cfg.under_shoot_pct = rc_cfg.undershoot_pct; oxcf->rc_cfg.over_shoot_pct = rc_cfg.overshoot_pct; + oxcf->rc_cfg.drop_frames_water_mark = rc_cfg.frame_drop_thresh; + rc->max_consec_drop = rc_cfg.max_consec_drop; oxcf->rc_cfg.max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct; oxcf->rc_cfg.max_inter_bitrate_pct = rc_cfg.max_inter_bitrate_pct; cpi_->framerate = rc_cfg.framerate; @@ -226,7 +231,8 @@ return true; } -void AV1RateControlRTC::ComputeQP(const AV1FrameParamsRTC &frame_params) { +FrameDropDecision AV1RateControlRTC::ComputeQP( + const AV1FrameParamsRTC &frame_params) { AV1_COMMON *const cm = &cpi_->common; int width, height; GF_GROUP *const gf_group = &cpi_->ppi->gf_group; @@ -292,14 +298,25 @@ } } av1_rc_set_frame_target(cpi_, target, cm->width, cm->height); - + // Always drop for spatial enhancement layer if layer bandwidth is 0. + // Otherwise check for frame-dropping based on buffer level in + // av1_rc_drop_frame(). + if ((cpi_->svc.spatial_layer_id > 0 && + cpi_->oxcf.rc_cfg.target_bandwidth == 0) || + av1_rc_drop_frame(cpi_)) { + cpi_->is_dropped_frame = true; + av1_rc_postencode_update_drop_frame(cpi_); + cpi_->frame_index_set.show_frame_count++; + cpi_->common.current_frame.frame_number++; + return FrameDropDecision::kDrop; + } int bottom_index, top_index; cpi_->common.quant_params.base_qindex = av1_rc_pick_q_and_bounds(cpi_, cm->width, cm->height, cpi_->gf_frame_index, &bottom_index, &top_index); - if (cpi_->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ) av1_cyclic_refresh_setup(cpi_); + return FrameDropDecision::kOk; } int AV1RateControlRTC::GetQP() const {
diff --git a/av1/ratectrl_rtc.h b/av1/ratectrl_rtc.h index a8b2deb..48ed9aa 100644 --- a/av1/ratectrl_rtc.h +++ b/av1/ratectrl_rtc.h
@@ -45,6 +45,8 @@ int overshoot_pct; int max_intra_bitrate_pct; int max_inter_bitrate_pct; + int frame_drop_thresh; + int max_consec_drop; double framerate; int layer_target_bitrate[kAV1MaxLayers]; int ts_rate_decimator[kAV1MaxTemporalLayers]; @@ -84,6 +86,11 @@ size_t delta_q_size; }; +enum class FrameDropDecision { + kOk, // Frame is encoded. + kDrop, // Frame is dropped. +}; + class AV1RateControlRTC { public: static std::unique_ptr<AV1RateControlRTC> Create( @@ -99,7 +106,11 @@ AV1CdefInfo GetCdefInfo() const; // Returns the segmentation map used for cyclic refresh, based on 4x4 blocks. bool GetSegmentationData(AV1SegmentationData *segmentation_data) const; - void ComputeQP(const AV1FrameParamsRTC &frame_params); + // ComputeQP returns the QP if the frame is not dropped (kOk return), + // otherwise it returns kDrop and subsequent GetQP and PostEncodeUpdate + // are not to be called (av1_rc_postencode_update_drop_frame is already + // called via ComputeQP if drop is decided). + FrameDropDecision ComputeQP(const AV1FrameParamsRTC &frame_params); // Feedback to rate control with the size of current encoded frame void PostEncodeUpdate(uint64_t encoded_frame_size);
diff --git a/test/ratectrl_rtc_test.cc b/test/ratectrl_rtc_test.cc index 04e90c8..92b7d39 100644 --- a/test/ratectrl_rtc_test.cc +++ b/test/ratectrl_rtc_test.cc
@@ -36,7 +36,8 @@ RcInterfaceTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), encoder_exit_(false), layer_frame_cnt_(0), superframe_cnt_(0), - dynamic_temporal_layers_(false), dynamic_spatial_layers_(false) { + dynamic_temporal_layers_(false), dynamic_spatial_layers_(false), + num_drops_(0), max_consec_drop_(0), frame_drop_thresh_(0) { memset(&svc_params_, 0, sizeof(svc_params_)); memset(&layer_id_, 0, sizeof(layer_id_)); } @@ -61,6 +62,7 @@ encoder->Control(AOME_SET_MAX_INTRA_BITRATE_PCT, rc_cfg_.max_intra_bitrate_pct); if (use_svc) encoder->Control(AV1E_SET_SVC_PARAMS, &svc_params_); + encoder->Control(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, max_consec_drop_); } // SVC specific settings if (use_svc) { @@ -144,16 +146,19 @@ superframe_cnt_++; int qp; encoder->Control(AOME_GET_LAST_QUANTIZER, &qp); - rc_api_->ComputeQP(frame_params_); - ASSERT_EQ(rc_api_->GetQP(), qp); - int encoder_lpf_level; - encoder->Control(AOME_GET_LOOPFILTER_LEVEL, &encoder_lpf_level); - aom::AV1LoopfilterLevel loopfilter_level = rc_api_->GetLoopfilterLevel(); - ASSERT_EQ(loopfilter_level.filter_level[0], encoder_lpf_level); - aom::AV1CdefInfo cdef_level = rc_api_->GetCdefInfo(); - int cdef_y_strengths[16]; - encoder->Control(AV1E_GET_LUMA_CDEF_STRENGTH, cdef_y_strengths); - ASSERT_EQ(cdef_level.cdef_strength_y, cdef_y_strengths[0]); + if (rc_api_->ComputeQP(frame_params_) == aom::FrameDropDecision::kOk) { + ASSERT_EQ(rc_api_->GetQP(), qp); + int encoder_lpf_level; + encoder->Control(AOME_GET_LOOPFILTER_LEVEL, &encoder_lpf_level); + aom::AV1LoopfilterLevel loopfilter_level = rc_api_->GetLoopfilterLevel(); + ASSERT_EQ(loopfilter_level.filter_level[0], encoder_lpf_level); + aom::AV1CdefInfo cdef_level = rc_api_->GetCdefInfo(); + int cdef_y_strengths[16]; + encoder->Control(AV1E_GET_LUMA_CDEF_STRENGTH, cdef_y_strengths); + ASSERT_EQ(cdef_level.cdef_strength_y, cdef_y_strengths[0]); + } else { + num_drops_++; + } } void FramePktHook(const aom_codec_cx_pkt_t *pkt) override { @@ -181,6 +186,27 @@ ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); } + void RunOneLayerDropFramesCBR() { + key_interval_ = 10000; + max_consec_drop_ = 8; + frame_drop_thresh_ = 30; + SetConfig(); + rc_cfg_.target_bandwidth = 100; + cfg_.rc_target_bitrate = 100; + rc_cfg_.max_quantizer = 50; + cfg_.rc_max_quantizer = 50; + rc_api_ = aom::AV1RateControlRTC::Create(rc_cfg_); + frame_params_.spatial_layer_id = 0; + frame_params_.temporal_layer_id = 0; + + ::libaom_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30, + 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Check that some frames were dropped, otherwise test has no value. + ASSERT_GE(num_drops_, 1); + } + void RunOneLayerPeriodicKey() { key_interval_ = 100; SetConfig(); @@ -270,6 +296,8 @@ rc_cfg_.max_quantizers[0] = 52; rc_cfg_.min_quantizers[0] = 2; rc_cfg_.aq_mode = aq_mode_; + rc_cfg_.frame_drop_thresh = frame_drop_thresh_; + rc_cfg_.max_consec_drop = max_consec_drop_; // Encoder settings for ground truth. cfg_.g_w = 640; @@ -288,6 +316,7 @@ cfg_.rc_target_bitrate = 1000; cfg_.kf_min_dist = key_interval_; cfg_.kf_max_dist = key_interval_; + cfg_.rc_dropframe_thresh = frame_drop_thresh_; } void SetConfigSvc(int number_spatial_layers, int number_temporal_layers) { @@ -427,10 +456,15 @@ int superframe_cnt_; bool dynamic_temporal_layers_; bool dynamic_spatial_layers_; + int num_drops_; + int max_consec_drop_; + int frame_drop_thresh_; }; TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); } +TEST_P(RcInterfaceTest, OneLayerDropFramesCBR) { RunOneLayerDropFramesCBR(); } + TEST_P(RcInterfaceTest, OneLayerPeriodicKey) { RunOneLayerPeriodicKey(); } TEST_P(RcInterfaceTest, Svc) { RunSvc(); }