[NORMATIVE] S_FRAME handling update This patch adds S-Frame implementation in the encoder, and fixes/improves the functionality of S-Frames in the decoder. Use the --sframe-dist=<arg> and --sframe-mode=<arg> switches to encode using S-Frames. *This patch allows combined inter prediction for S-Frames, in contrast with the specification in the initial design document.* BUG=aomedia:1459 Change-Id: Ia8b9dc9f7b97fcd435e79c62f2adc0e01f557627
diff --git a/aom/aom_encoder.h b/aom/aom_encoder.h index 67e718f..e93d081 100644 --- a/aom/aom_encoder.h +++ b/aom/aom_encoder.h
@@ -626,6 +626,28 @@ */ unsigned int kf_max_dist; + /*!\brief sframe interval + * + * This value, expressed as a number of frames, forces the encoder to code + * an S-Frame every sframe_dist frames. + */ + unsigned int sframe_dist; + + /*!\brief sframe insertion mode + * + * This value must be set to 1 or 2, and tells the encoder how to insert + * S-Frames. It will only have an effect if sframe_dist != 0. + * + * If altref is enabled: + * - if sframe_mode == 1, the considered frame will be made into an + * S-Frame only if it is an altref frame + * - if sframe_mode == 2, the next altref frame will be made into an + * S-Frame. + * + * Otherwise: the considered frame will be made into an S-Frame. + */ + unsigned int sframe_mode; + /*!\brief Tile coding mode * * This value indicates the tile coding mode.
diff --git a/aomenc.c b/aomenc.c index c1dec26..7276bc5 100644 --- a/aomenc.c +++ b/aomenc.c
@@ -383,7 +383,10 @@ ARG_DEF(NULL, "disable-kf", 0, "Disable keyframe placement"); static const arg_def_t *kf_args[] = { &kf_min_dist, &kf_max_dist, &kf_disabled, NULL }; - +static const arg_def_t sframe_dist = + ARG_DEF(NULL, "sframe-dist", 1, "S-Frame interval (frames)"); +static const arg_def_t sframe_mode = + ARG_DEF(NULL, "sframe-mode", 1, "S-Frame insertion mode (1..2)"); static const arg_def_t noise_sens = ARG_DEF(NULL, "noise-sensitivity", 1, "Noise sensitivity (frames to blur)"); static const arg_def_t sharpness = @@ -687,6 +690,8 @@ &disable_tempmv, &bitdeptharg, &inbitdeptharg, + &sframe_dist, + &sframe_mode, NULL }; static const int av1_arg_ctrl_map[] = { AOME_SET_CPUUSED, AOME_SET_DEVSF, @@ -1275,6 +1280,10 @@ config->cfg.kf_max_dist = arg_parse_uint(&arg); } else if (arg_match(&arg, &kf_disabled, argi)) { config->cfg.kf_mode = AOM_KF_DISABLED; + } else if (arg_match(&arg, &sframe_dist, argi)) { + config->cfg.sframe_dist = arg_parse_uint(&arg); + } else if (arg_match(&arg, &sframe_mode, argi)) { + config->cfg.sframe_mode = arg_parse_uint(&arg); #if CONFIG_MAX_TILE } else if (arg_match(&arg, &tile_width, argi)) { config->cfg.tile_width_count =
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c index d3ef1e8..354464d 100644 --- a/av1/av1_cx_iface.c +++ b/av1/av1_cx_iface.c
@@ -597,7 +597,9 @@ cfg->kf_mode == AOM_KF_AUTO && cfg->kf_min_dist != cfg->kf_max_dist; oxcf->key_freq = cfg->kf_max_dist; - + oxcf->sframe_dist = cfg->sframe_dist; + oxcf->sframe_mode = cfg->sframe_mode; + oxcf->sframe_enabled = cfg->sframe_dist != 0; oxcf->speed = extra_cfg->cpu_used; oxcf->dev_sf = extra_cfg->dev_sf; oxcf->enable_auto_arf = extra_cfg->enable_auto_alt_ref; @@ -1763,6 +1765,8 @@ AOM_KF_AUTO, // g_kfmode 0, // kf_min_dist 9999, // kf_max_dist + 0, // sframe_dist + 1, // sframe_mode 0, // large_scale_tile 0, // monochrome 0, // tile_width_count
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 4d0720e..d556188 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -1053,7 +1053,7 @@ static INLINE MOTION_MODE motion_mode_allowed(const WarpedMotionParams *gm_params, const MACROBLOCKD *xd, - const MODE_INFO *mi) { + const MODE_INFO *mi, int can_use_previous) { const MB_MODE_INFO *mbmi = &mi->mbmi; #if CONFIG_AMVR if (xd->cur_frame_force_integer_mv == 0) { @@ -1069,7 +1069,7 @@ if (!check_num_overlappable_neighbors(mbmi)) return SIMPLE_TRANSLATION; assert(!has_second_ref(mbmi)); if (mbmi->num_proj_ref[0] >= 1 && - !av1_is_scaled(&(xd->block_refs[0]->sf))) { + (can_use_previous && !av1_is_scaled(&(xd->block_refs[0]->sf)))) { #if CONFIG_AMVR if (xd->cur_frame_force_integer_mv) { return OBMC_CAUSAL; @@ -1086,9 +1086,10 @@ static INLINE void assert_motion_mode_valid(MOTION_MODE mode, const WarpedMotionParams *gm_params, const MACROBLOCKD *xd, - const MODE_INFO *mi) { + const MODE_INFO *mi, + int can_use_previous) { const MOTION_MODE last_motion_mode_allowed = - motion_mode_allowed(gm_params, xd, mi); + motion_mode_allowed(gm_params, xd, mi, can_use_previous); // Check that the input mode is not illegal if (last_motion_mode_allowed < mode)
diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h index 76c5eea..fa233e6 100644 --- a/av1/common/onyxc_int.h +++ b/av1/common/onyxc_int.h
@@ -637,6 +637,10 @@ return cm->frame_type == KEY_FRAME || cm->intra_only; } +static INLINE int frame_is_sframe(const AV1_COMMON *cm) { + return cm->frame_type == S_FRAME; +} + static INLINE RefCntBuffer *get_prev_frame(const AV1_COMMON *const cm) { if (cm->primary_ref_frame == PRIMARY_REF_NONE || cm->frame_refs[cm->primary_ref_frame].idx == INVALID_IDX) { @@ -646,6 +650,7 @@ ->frame_bufs[cm->frame_refs[cm->primary_ref_frame].idx]; } } + // Returns 1 if this frame might use mvs from some previous frame. This // function doesn't consider whether prev_frame is actually suitable (see // frame_can_use_prev_frame_mvs for that)
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c index 1bb9ff5..921b91b 100644 --- a/av1/common/reconinter.c +++ b/av1/common/reconinter.c
@@ -73,11 +73,12 @@ int w, int h, ConvolveParams *conv_params, InterpFilters interp_filters, const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, int ref, const MODE_INFO *mi, int build_for_obmc, int xs, int ys, - const MACROBLOCKD *xd) { + const MACROBLOCKD *xd, int can_use_previous) { (void)xd; // Make sure the selected motion mode is valid for this configuration - assert_motion_mode_valid(mi->mbmi.motion_mode, xd->global_motion, xd, mi); + assert_motion_mode_valid(mi->mbmi.motion_mode, xd->global_motion, xd, mi, + can_use_previous); assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL)); @@ -718,7 +719,7 @@ const int subpel_x, const int subpel_y, const struct scale_factors *sf, int w, int h, ConvolveParams *conv_params, InterpFilters interp_filters, int xs, int ys, int plane, const WarpTypesAllowed *warp_types, int p_col, - int p_row, int ref, MACROBLOCKD *xd) { + int p_row, int ref, MACROBLOCKD *xd, int can_use_previous) { const MODE_INFO *mi = xd->mi[0]; (void)dst; (void)dst_stride; @@ -757,7 +758,7 @@ av1_make_inter_predictor(pre, pre_stride, tmp_dst, MAX_SB_SIZE, subpel_x, subpel_y, sf, w, h, conv_params, interp_filters, warp_types, p_col, p_row, plane, ref, mi, 0, xs, ys, - xd); + xd, can_use_previous); if (!plane && comp_data.interinter_compound_type == COMPOUND_SEG) { build_compound_seg_mask_d32( @@ -778,7 +779,7 @@ const MV *src_mv, const struct scale_factors *sf, int w, int h, int ref, InterpFilters interp_filters, const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, enum mv_precision precision, int x, int y, - const MACROBLOCKD *xd) { + const MACROBLOCKD *xd, int can_use_previous) { const int is_q4 = precision == MV_PRECISION_Q4; const MV32 mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, is_q4 ? src_mv->col : src_mv->col * 2 }; @@ -795,15 +796,18 @@ av1_make_inter_predictor(src, src_stride, dst, dst_stride, subpel_x, subpel_y, sf, w, h, &conv_params, interp_filters, warp_types, p_col, p_row, plane, ref, xd->mi[0], 0, - sf->x_step_q4, sf->y_step_q4, xd); + sf->x_step_q4, sf->y_step_q4, xd, can_use_previous); } -void av1_build_inter_predictor( - const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, - const MV *src_mv, const struct scale_factors *sf, int w, int h, - ConvolveParams *conv_params, InterpFilters interp_filters, - const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, - int ref, enum mv_precision precision, int x, int y, const MACROBLOCKD *xd) { +void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, const MV *src_mv, + const struct scale_factors *sf, int w, int h, + ConvolveParams *conv_params, + InterpFilters interp_filters, + const WarpTypesAllowed *warp_types, int p_col, + int p_row, int plane, int ref, + enum mv_precision precision, int x, int y, + const MACROBLOCKD *xd, int can_use_previous) { const int is_q4 = precision == MV_PRECISION_Q4; const MV32 mv_q4 = { is_q4 ? src_mv->row : src_mv->row * 2, is_q4 ? src_mv->col : src_mv->col * 2 }; @@ -819,7 +823,7 @@ av1_make_inter_predictor(src, src_stride, dst, dst_stride, subpel_x, subpel_y, sf, w, h, conv_params, interp_filters, warp_types, p_col, p_row, plane, ref, xd->mi[0], 0, - sf->x_step_q4, sf->y_step_q4, xd); + sf->x_step_q4, sf->y_step_q4, xd, can_use_previous); } typedef struct SubpelParams { @@ -1041,7 +1045,7 @@ pre, pre_buf->stride, dst, dst_buf->stride, subpel_x, subpel_y, sf, b4_w, b4_h, &conv_params, this_mbmi->interp_filters, &warp_types, (mi_x >> pd->subsampling_x) + x, (mi_y >> pd->subsampling_y) + y, - plane, ref, mi, build_for_obmc, xs, ys, xd); + plane, ref, mi, build_for_obmc, xs, ys, xd, cm->use_ref_frame_mvs); ++col; } @@ -1144,7 +1148,7 @@ &conv_params, mi->mbmi.interp_filters, subpel_params[ref].xs, subpel_params[ref].ys, plane, &warp_types, (mi_x >> pd->subsampling_x) + x, (mi_y >> pd->subsampling_y) + y, - ref, xd); + ref, xd, cm->use_ref_frame_mvs); else av1_make_inter_predictor( pre[ref], pre_buf->stride, dst, dst_buf->stride, @@ -1152,7 +1156,7 @@ &conv_params, mi->mbmi.interp_filters, &warp_types, (mi_x >> pd->subsampling_x) + x, (mi_y >> pd->subsampling_y) + y, plane, ref, mi, build_for_obmc, subpel_params[ref].xs, - subpel_params[ref].ys, xd); + subpel_params[ref].ys, xd, cm->use_ref_frame_mvs); } // TODO(angiebird): This part needs optimization @@ -1901,7 +1905,8 @@ int bw, int bh, int x, int y, int w, int h, int mi_x, int mi_y, int ref, uint8_t *const ext_dst, - int ext_dst_stride) { + int ext_dst_stride, + int can_use_previous) { struct macroblockd_plane *const pd = &xd->plane[plane]; const MODE_INFO *mi = xd->mi[0]; @@ -1962,12 +1967,13 @@ pre, pre_buf->stride, dst, ext_dst_stride, subpel_x, subpel_y, sf, w, h, &conv_params, mi->mbmi.interp_filters, &warp_types, (mi_x >> pd->subsampling_x) + x, (mi_y >> pd->subsampling_y) + y, plane, - ref, mi, 0, xs, ys, xd); + ref, mi, 0, xs, ys, xd, can_use_previous); } void av1_build_inter_predictors_for_planes_single_buf( MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, int mi_row, - int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3]) { + int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3], + int can_use_previous) { int plane; const int mi_x = mi_col * MI_SIZE; const int mi_y = mi_row * MI_SIZE; @@ -1978,7 +1984,7 @@ const int bh = block_size_high[plane_bsize]; build_inter_predictors_single_buf(xd, plane, bw, bh, 0, 0, bw, bh, mi_x, mi_y, ref, ext_dst[plane], - ext_dst_stride[plane]); + ext_dst_stride[plane], can_use_previous); } }
diff --git a/av1/common/reconinter.h b/av1/common/reconinter.h index 694ed4c..d5475d3 100644 --- a/av1/common/reconinter.h +++ b/av1/common/reconinter.h
@@ -195,7 +195,7 @@ const int subpel_x, const int subpel_y, const struct scale_factors *sf, int w, int h, ConvolveParams *conv_params, InterpFilters interp_filters, int xs, int ys, int plane, const WarpTypesAllowed *warp_types, int p_col, - int p_row, int ref, MACROBLOCKD *xd); + int p_row, int ref, MACROBLOCKD *xd, int can_use_previous); // TODO(jkoleszar): yet another mv clamping function :-( static INLINE MV32 clamp_mv_to_umv_border_sb(const MACROBLOCKD *xd, @@ -233,19 +233,22 @@ int mi_row, int mi_col, BUFFER_SET *ctx, BLOCK_SIZE bsize); -void av1_build_inter_predictor( - const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, - const MV *src_mv, const struct scale_factors *sf, int w, int h, - ConvolveParams *conv_params, InterpFilters interp_filters, - const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, - int ref, enum mv_precision precision, int x, int y, const MACROBLOCKD *xd); +void av1_build_inter_predictor(const uint8_t *src, int src_stride, uint8_t *dst, + int dst_stride, const MV *src_mv, + const struct scale_factors *sf, int w, int h, + ConvolveParams *conv_params, + InterpFilters interp_filters, + const WarpTypesAllowed *warp_types, int p_col, + int p_row, int plane, int ref, + enum mv_precision precision, int x, int y, + const MACROBLOCKD *xd, int can_use_previous); void av1_highbd_build_inter_predictor( const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, const MV *mv_q3, const struct scale_factors *sf, int w, int h, int do_avg, InterpFilters interp_filters, const WarpTypesAllowed *warp_types, int p_col, int p_row, int plane, enum mv_precision precision, int x, int y, - const MACROBLOCKD *xd); + const MACROBLOCKD *xd, int can_use_previous); static INLINE int scaled_buffer_offset(int x_offset, int y_offset, int stride, const struct scale_factors *sf) { @@ -414,7 +417,8 @@ // Encoder only void av1_build_inter_predictors_for_planes_single_buf( MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, int mi_row, - int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3]); + int mi_col, int ref, uint8_t *ext_dst[3], int ext_dst_stride[3], + int can_use_previous); void av1_build_wedge_inter_predictor_from_buf(MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane_from, int plane_to, uint8_t *ext_dst0[3],
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 2d107bd..85cea50 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -2709,7 +2709,7 @@ cm->frame_type = (FRAME_TYPE)aom_rb_read_literal(rb, 2); // 2 bits cm->show_frame = aom_rb_read_bit(rb); cm->intra_only = cm->frame_type == INTRA_ONLY_FRAME; - cm->error_resilient_mode = aom_rb_read_bit(rb); + cm->error_resilient_mode = frame_is_sframe(cm) ? 1 : aom_rb_read_bit(rb); cm->enable_intra_edge_filter = aom_rb_read_bit(rb); cm->allow_filter_intra = aom_rb_read_bit(rb); @@ -2776,7 +2776,8 @@ } } - int frame_size_override_flag = aom_rb_read_literal(rb, 1); + int frame_size_override_flag = + frame_is_sframe(cm) ? 1 : aom_rb_read_literal(rb, 1); cm->allow_intrabc = 0; #if CONFIG_FRAME_REFS_SIGNALING @@ -2876,9 +2877,8 @@ (av1_superres_unscaled(cm) || !NO_FILTER_FOR_IBC)) cm->allow_intrabc = aom_rb_read_bit(rb); } else if (pbi->need_resync != 1) { /* Skip if need resync */ - pbi->refresh_frame_flags = (cm->frame_type == S_FRAME) - ? 0xFF - : aom_rb_read_literal(rb, REF_FRAMES); + pbi->refresh_frame_flags = + frame_is_sframe(cm) ? 0xFF : aom_rb_read_literal(rb, REF_FRAMES); if (!pbi->refresh_frame_flags) { // NOTE: "pbi->refresh_frame_flags == 0" indicates that the coded frame @@ -2966,7 +2966,7 @@ } } - if (cm->error_resilient_mode == 0 && frame_size_override_flag) { + if (!cm->error_resilient_mode && frame_size_override_flag) { setup_frame_size_with_refs(cm, rb); } else { setup_frame_size(cm, frame_size_override_flag, rb); @@ -3013,7 +3013,7 @@ av1_setup_frame_buf_refs(cm); - if (cm->frame_type != S_FRAME) av1_setup_frame_sign_bias(cm); + if (!frame_is_sframe(cm)) av1_setup_frame_sign_bias(cm); cm->cur_frame->intra_only = cm->frame_type == KEY_FRAME || cm->intra_only; cm->cur_frame->frame_type = cm->frame_type; @@ -3102,7 +3102,6 @@ setup_quantization(cm, rb); xd->bd = (int)cm->bit_depth; - if (frame_is_intra_only(cm) || cm->error_resilient_mode) { av1_setup_past_independence(cm); av1_setup_frame_contexts(cm);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 7b62c96..cf09814 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -228,7 +228,7 @@ if (mbmi->skip_mode) return SIMPLE_TRANSLATION; const MOTION_MODE last_motion_mode_allowed = - motion_mode_allowed(xd->global_motion, xd, mi); + motion_mode_allowed(xd->global_motion, xd, mi, cm->use_ref_frame_mvs); int motion_mode; if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return SIMPLE_TRANSLATION;
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 108230d..afe2d9d 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -270,7 +270,8 @@ MOTION_MODE last_motion_mode_allowed = cm->switchable_motion_mode - ? motion_mode_allowed(cm->global_motion, xd, mi) + ? motion_mode_allowed(cm->global_motion, xd, mi, + cm->use_ref_frame_mvs) : SIMPLE_TRANSLATION; assert(mbmi->motion_mode <= last_motion_mode_allowed); switch (last_motion_mode_allowed) { @@ -2351,6 +2352,9 @@ #if USE_GF16_MULTI_LAYER static int get_refresh_mask_gf16(AV1_COMP *cpi) { + if (cpi->common.frame_type == KEY_FRAME || frame_is_sframe(&cpi->common)) + return 0xFF; + int refresh_mask = 0; if (cpi->refresh_last_frame || cpi->refresh_golden_frame || @@ -2365,6 +2369,9 @@ #endif // USE_GF16_MULTI_LAYER static int get_refresh_mask(AV1_COMP *cpi) { + if (cpi->common.frame_type == KEY_FRAME || frame_is_sframe(&cpi->common)) + return 0xFF; + int refresh_mask = 0; #if USE_GF16_MULTI_LAYER if (cpi->rc.baseline_gf_interval == 16) return get_refresh_mask_gf16(cpi); @@ -2992,7 +2999,11 @@ if (cm->intra_only) cm->frame_type = INTRA_ONLY_FRAME; aom_wb_write_bit(wb, cm->show_frame); - aom_wb_write_bit(wb, cm->error_resilient_mode); + if (frame_is_sframe(cm)) { + assert(cm->error_resilient_mode); + } else { + aom_wb_write_bit(wb, cm->error_resilient_mode); + } aom_wb_write_bit(wb, cm->enable_intra_edge_filter); aom_wb_write_bit(wb, cm->allow_filter_intra); @@ -3031,10 +3042,12 @@ aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM, "Frame dimensions are larger than the maximum values"); } + int frame_size_override_flag = - (cm->width != cm->seq_params.max_frame_width || - cm->height != cm->seq_params.max_frame_height); - aom_wb_write_bit(wb, frame_size_override_flag); + frame_is_sframe(cm) ? 1 + : (cm->width != cm->seq_params.max_frame_width || + cm->height != cm->seq_params.max_frame_height); + if (!frame_is_sframe(cm)) aom_wb_write_bit(wb, frame_size_override_flag); #if CONFIG_FRAME_REFS_SIGNALING cm->frame_refs_short_signaling = 0; @@ -3108,12 +3121,14 @@ (av1_superres_unscaled(cm) || !NO_FILTER_FOR_IBC)) aom_wb_write_bit(wb, cm->allow_intrabc); } - } else if (cm->frame_type == INTER_FRAME || cm->frame_type == S_FRAME) { + } else if (cm->frame_type == INTER_FRAME || frame_is_sframe(cm)) { MV_REFERENCE_FRAME ref_frame; + cpi->refresh_frame_mask = get_refresh_mask(cpi); if (cm->frame_type == INTER_FRAME) { - cpi->refresh_frame_mask = get_refresh_mask(cpi); aom_wb_write_literal(wb, cpi->refresh_frame_mask, REF_FRAMES); + } else { + assert(frame_is_sframe(cm) && cpi->refresh_frame_mask == 0xFF); } int updated_fb = -1; @@ -3162,7 +3177,7 @@ #endif // CONFIG_FRAME_REFS_SIGNALING aom_wb_write_literal(wb, get_ref_frame_map_idx(cpi, ref_frame), REF_FRAMES_LOG2); - if (cm->frame_type == S_FRAME) { + if (frame_is_sframe(cm)) { assert(cm->ref_frame_sign_bias[ref_frame] == 0); } @@ -3182,7 +3197,7 @@ } } - if (cm->error_resilient_mode == 0 && frame_size_override_flag) { + if (!cm->error_resilient_mode && frame_size_override_flag) { write_frame_size_with_refs(cpi, wb); } else { write_frame_size(cm, frame_size_override_flag, wb); @@ -3207,10 +3222,8 @@ } } - if (cm->seq_params.frame_id_numbers_present_flag) { - cm->refresh_mask = - cm->frame_type == KEY_FRAME ? 0xFF : get_refresh_mask(cpi); - } + if (cm->seq_params.frame_id_numbers_present_flag) + cm->refresh_mask = get_refresh_mask(cpi); const int might_bwd_adapt = !(cm->large_scale_tile); if (might_bwd_adapt) {
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index fd841a8..b6a927c 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -1221,7 +1221,8 @@ set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]); const MOTION_MODE motion_allowed = cm->switchable_motion_mode - ? motion_mode_allowed(xd->global_motion, xd, mi) + ? motion_mode_allowed(xd->global_motion, xd, mi, + cm->use_ref_frame_mvs) : SIMPLE_TRANSLATION; if (mbmi->ref_frame[1] != INTRA_FRAME) { if (motion_allowed == WARPED_CAUSAL) {
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index 8efcd14..e3741a4 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -341,6 +341,11 @@ set_sb_size(&cm->seq_params, select_sb_size(cpi)); set_use_reference_buffer(cm, 0); cm->pre_fc = &cm->frame_contexts[FRAME_CONTEXT_DEFAULTS]; + } else if (frame_is_sframe(cm)) { + cpi->refresh_golden_frame = 1; + cpi->refresh_alt_ref_frame = 1; + av1_zero(cpi->interp_filter_selected); + set_sb_size(&cm->seq_params, select_sb_size(cpi)); } else { if (cm->primary_ref_frame == PRIMARY_REF_NONE || cm->frame_refs[cm->primary_ref_frame].idx < 0) { @@ -3513,7 +3518,7 @@ BufferPool *const pool = cm->buffer_pool; // At this point the new frame has been encoded. // If any buffer copy / swapping is signaled it should be done here. - if (cm->frame_type == KEY_FRAME) { + if (cm->frame_type == KEY_FRAME || frame_is_sframe(cm)) { ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->gld_fb_idx], cm->new_fb_idx); ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->bwd_fb_idx], @@ -3640,7 +3645,7 @@ // lst_fb_idxes[2], lst_fb_idxes[0], lst_fb_idxes[1] int ref_frame; - if (cm->frame_type == KEY_FRAME) { + if (cm->frame_type == KEY_FRAME || frame_is_sframe(cm)) { for (ref_frame = 0; ref_frame < LAST_REF_FRAMES; ++ref_frame) { ref_cnt_fb(pool->frame_bufs, &cm->ref_frame_map[cpi->lst_fb_idxes[ref_frame]], @@ -4978,7 +4983,7 @@ cpi->sf.interp_filter_search_mask = setup_interp_filter_search_mask(cpi); // Set various flags etc to special state if it is a key frame. - if (frame_is_intra_only(cm)) { + if (frame_is_intra_only(cm) || frame_is_sframe(cm)) { // Reset the loop filter deltas and segmentation map. av1_reset_segment_features(cm); @@ -4990,7 +4995,9 @@ // The alternate reference frame cannot be active for a key frame. cpi->rc.source_alt_ref_active = 0; - cm->error_resilient_mode = oxcf->error_resilient_mode; + // S_FRAMEs are always error resilient + cm->error_resilient_mode = + oxcf->error_resilient_mode || frame_is_sframe(cm); } if (cpi->oxcf.mtu == 0) { cm->num_tg = cpi->oxcf.num_tile_groups; @@ -5038,6 +5045,13 @@ msb = cpi->source->y_buffer[1] & 0xff; } cm->current_frame_id = ((msb << 8) + lsb) % (1 << frame_id_length); + + // S_frame is meant for stitching different streams of different + // resolutions together, so current_frame_id must be the + // same across different streams of the same content current_frame_id + // should be the same and not random. 0x37 is a chosen number as start + // point + if (cpi->oxcf.sframe_enabled) cm->current_frame_id = 0x37; } else { cm->current_frame_id = (cm->current_frame_id + 1 + (1 << frame_id_length)) % @@ -5093,8 +5107,8 @@ } } - // If the encoder forced a KEY_FRAME decision - if (cm->frame_type == KEY_FRAME) { + // If the encoder forced a KEY_FRAME decision or if frame is an S_FRAME + if (cm->frame_type == KEY_FRAME || frame_is_sframe(cm)) { cpi->refresh_last_frame = 1; }
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index c33c9ed..7a32c77 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -161,7 +161,9 @@ // Key Framing Operations int auto_key; // autodetect cut scenes and set the keyframes int key_freq; // maximum distance to key frame. - + int sframe_dist; + int sframe_mode; + int sframe_enabled; int lag_in_frames; // how many frames lag before we start encoding // ----------------------------------------------------------------
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c index 523ffa1..ed68c34 100644 --- a/av1/encoder/ratectrl.c +++ b/av1/encoder/ratectrl.c
@@ -1340,6 +1340,10 @@ AV1_COMMON *const cm = &cpi->common; RATE_CONTROL *const rc = &cpi->rc; int target; + int altref_enabled = is_altref_enabled(cpi); + int sframe_dist = cpi->oxcf.sframe_dist; + int sframe_mode = cpi->oxcf.sframe_mode; + int sframe_enabled = cpi->oxcf.sframe_enabled; // TODO(yaowu): replace the "auto_key && 0" below with proper decision logic. if (!cpi->refresh_alt_ref_frame && (cm->current_video_frame == 0 || (cpi->frame_flags & FRAMEFLAGS_KEY) || @@ -1352,6 +1356,37 @@ rc->source_alt_ref_active = 0; } else { cm->frame_type = INTER_FRAME; + if (sframe_enabled) { + if (altref_enabled) { + if (sframe_mode == 1) { + // sframe_mode == 1: insert sframe if it matches altref frame. + + if (cm->current_video_frame % sframe_dist == 0 && + cm->frame_type != KEY_FRAME && cm->current_video_frame != 0 && + cpi->refresh_alt_ref_frame) { + cm->frame_type = S_FRAME; + } + } else { + // sframe_mode != 1: if sframe will be inserted at the next available + // altref frame + + if (cm->current_video_frame % sframe_dist == 0 && + cm->frame_type != KEY_FRAME && cm->current_video_frame != 0) { + rc->sframe_due = 1; + } + + if (rc->sframe_due && cpi->refresh_alt_ref_frame) { + cm->frame_type = S_FRAME; + rc->sframe_due = 0; + } + } + } else { + if (cm->current_video_frame % sframe_dist == 0 && + cm->frame_type != KEY_FRAME && cm->current_video_frame != 0) { + cm->frame_type = S_FRAME; + } + } + } } if (rc->frames_till_gf_update_due == 0) { rc->baseline_gf_interval = (rc->min_gf_interval + rc->max_gf_interval) / 2;
diff --git a/av1/encoder/ratectrl.h b/av1/encoder/ratectrl.h index af3632f..81157ce 100644 --- a/av1/encoder/ratectrl.h +++ b/av1/encoder/ratectrl.h
@@ -84,6 +84,7 @@ int source_alt_ref_pending; int source_alt_ref_active; int is_src_frame_alt_ref; + int sframe_due; // Length of the bi-predictive frame group interval int bipred_group_interval;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 60b453b..e94c797 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -5340,14 +5340,15 @@ ref_yv12[!id].buf, ref_yv12[!id].stride, second_pred, pw, &frame_mv[refs[!id]].as_mv, &cm->sf_identity, pw, ph, 0, interp_filters, &warp_types, p_col, p_row, plane, MV_PRECISION_Q3, - mi_col * MI_SIZE, mi_row * MI_SIZE, xd); + mi_col * MI_SIZE, mi_row * MI_SIZE, xd, cm->use_ref_frame_mvs); } else { second_pred = (uint8_t *)second_pred_alloc_16; - av1_build_inter_predictor( - ref_yv12[!id].buf, ref_yv12[!id].stride, second_pred, pw, - &frame_mv[refs[!id]].as_mv, &cm->sf_identity, pw, ph, &conv_params, - interp_filters, &warp_types, p_col, p_row, plane, !id, - MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd); + av1_build_inter_predictor(ref_yv12[!id].buf, ref_yv12[!id].stride, + second_pred, pw, &frame_mv[refs[!id]].as_mv, + &cm->sf_identity, pw, ph, &conv_params, + interp_filters, &warp_types, p_col, p_row, + plane, !id, MV_PRECISION_Q3, mi_col * MI_SIZE, + mi_row * MI_SIZE, xd, cm->use_ref_frame_mvs); } const int order_idx = id != 0; @@ -5919,12 +5920,14 @@ av1_highbd_build_inter_predictor( ref_yv12.buf, ref_yv12.stride, second_pred, pw, other_mv, &sf, pw, ph, 0, mbmi->interp_filters, &warp_types, p_col, p_row, plane, - MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd); + MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd, + cm->use_ref_frame_mvs); } else { av1_build_inter_predictor( ref_yv12.buf, ref_yv12.stride, second_pred, pw, other_mv, &sf, pw, ph, &conv_params, mbmi->interp_filters, &warp_types, p_col, p_row, plane, - !ref_idx, MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd); + !ref_idx, MV_PRECISION_Q3, mi_col * MI_SIZE, mi_row * MI_SIZE, xd, + cm->use_ref_frame_mvs); } av1_jnt_comp_weight_assign(cm, mbmi, 0, &xd->jcp_param.fwd_offset, @@ -6879,7 +6882,8 @@ base_mbmi = *mbmi; MOTION_MODE last_motion_mode_allowed = cm->switchable_motion_mode - ? motion_mode_allowed(xd->global_motion, xd, mi) + ? motion_mode_allowed(xd->global_motion, xd, mi, + cm->use_ref_frame_mvs) : SIMPLE_TRANSLATION; assert(mbmi->ref_frame[1] != INTRA_FRAME); const MV_REFERENCE_FRAME ref_frame_1 = mbmi->ref_frame[1]; @@ -7626,6 +7630,7 @@ int tmp_rate_mv; COMPOUND_TYPE cur_type; int best_compmode_interinter_cost = 0; + int can_use_previous = cm->use_ref_frame_mvs; best_mv[0].as_int = cur_mv[0].as_int; best_mv[1].as_int = cur_mv[1].as_int; @@ -7636,9 +7641,11 @@ if (masked_compound_used) { // get inter predictors to use for masked compound modes av1_build_inter_predictors_for_planes_single_buf( - xd, bsize, 0, 0, mi_row, mi_col, 0, preds0, strides); + xd, bsize, 0, 0, mi_row, mi_col, 0, preds0, strides, + can_use_previous); av1_build_inter_predictors_for_planes_single_buf( - xd, bsize, 0, 0, mi_row, mi_col, 1, preds1, strides); + xd, bsize, 0, 0, mi_row, mi_col, 1, preds1, strides, + can_use_previous); } for (cur_type = COMPOUND_AVERAGE; cur_type < COMPOUND_TYPES; cur_type++) {
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c index 25dedc7..f87a996 100644 --- a/av1/encoder/temporal_filter.c +++ b/av1/encoder/temporal_filter.c
@@ -35,7 +35,8 @@ static void temporal_filter_predictors_mb_c( MACROBLOCKD *xd, uint8_t *y_mb_ptr, uint8_t *u_mb_ptr, uint8_t *v_mb_ptr, int stride, int uv_block_width, int uv_block_height, int mv_row, int mv_col, - uint8_t *pred, struct scale_factors *scale, int x, int y) { + uint8_t *pred, struct scale_factors *scale, int x, int y, + int can_use_previous) { const int which_mv = 0; const MV mv = { mv_row, mv_col }; enum mv_precision mv_precision_uv; @@ -55,34 +56,35 @@ } if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - av1_highbd_build_inter_predictor( - y_mb_ptr, stride, &pred[0], 16, &mv, scale, 16, 16, which_mv, - interp_filters, &warp_types, x, y, 0, MV_PRECISION_Q3, x, y, xd); + av1_highbd_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale, + 16, 16, which_mv, interp_filters, + &warp_types, x, y, 0, MV_PRECISION_Q3, x, + y, xd, can_use_previous); av1_highbd_build_inter_predictor( u_mb_ptr, uv_stride, &pred[256], uv_block_width, &mv, scale, uv_block_width, uv_block_height, which_mv, interp_filters, &warp_types, - x, y, 1, mv_precision_uv, x, y, xd); + x, y, 1, mv_precision_uv, x, y, xd, can_use_previous); av1_highbd_build_inter_predictor( v_mb_ptr, uv_stride, &pred[512], uv_block_width, &mv, scale, uv_block_width, uv_block_height, which_mv, interp_filters, &warp_types, - x, y, 2, mv_precision_uv, x, y, xd); + x, y, 2, mv_precision_uv, x, y, xd, can_use_previous); return; } av1_build_inter_predictor(y_mb_ptr, stride, &pred[0], 16, &mv, scale, 16, 16, &conv_params, interp_filters, &warp_types, x, y, 0, - 0, MV_PRECISION_Q3, x, y, xd); + 0, MV_PRECISION_Q3, x, y, xd, can_use_previous); av1_build_inter_predictor(u_mb_ptr, uv_stride, &pred[256], uv_block_width, &mv, scale, uv_block_width, uv_block_height, &conv_params, interp_filters, &warp_types, x, y, 1, - 0, mv_precision_uv, x, y, xd); + 0, mv_precision_uv, x, y, xd, can_use_previous); av1_build_inter_predictor(v_mb_ptr, uv_stride, &pred[512], uv_block_width, &mv, scale, uv_block_width, uv_block_height, &conv_params, interp_filters, &warp_types, x, y, 2, - 0, mv_precision_uv, x, y, xd); + 0, mv_precision_uv, x, y, xd, can_use_previous); } void av1_temporal_filter_apply_c(uint8_t *frame1, unsigned int stride, @@ -385,7 +387,7 @@ frames[frame]->v_buffer + mb_uv_offset, frames[frame]->y_stride, mb_uv_width, mb_uv_height, mbd->mi[0]->mbmi.mv[0].as_mv.row, mbd->mi[0]->mbmi.mv[0].as_mv.col, predictor, scale, mb_col * 16, - mb_row * 16); + mb_row * 16, cm->use_ref_frame_mvs); // Apply the filter (YUV) if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {