rtc: Allow for using golden reference in superres mode
Fix some conditions to allow superres mode in RTC
to use the golden reference: allow for scaled
golden in variance partitioning and av1_mv_pred
on last-ref in nonrd_pickmode.
And only release golden scaled_ref if its refreshed,
for non-svc mode. This allows to only scale the golden
reference it if is was last refreshed on previous frame.
This avoids additional reference frame scaling on every
frame, for rtc mode with superres.
Change-Id: I7c51e9e6a39b79f5889f8b6d4d81b47d20ee7a30
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index e282dc1..550504f 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2581,12 +2581,15 @@
// av1_scale_references. Note GOLDEN is forced to update on the (first/tigger)
// resized frame and ALTREF will be refreshed ~4 frames later, so both
// references become available again after few frames.
+ // For superres: don't disable golden reference.
if (svc->number_spatial_layers == 1) {
- if (cpi->ref_frame_flags & av1_ref_frame_flag_list[GOLDEN_FRAME]) {
- const YV12_BUFFER_CONFIG *const ref =
- get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
- if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height)
- cpi->ref_frame_flags ^= AOM_GOLD_FLAG;
+ if (!cpi->oxcf.superres_cfg.enable_superres) {
+ if (cpi->ref_frame_flags & av1_ref_frame_flag_list[GOLDEN_FRAME]) {
+ const YV12_BUFFER_CONFIG *const ref =
+ get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
+ if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height)
+ cpi->ref_frame_flags ^= AOM_GOLD_FLAG;
+ }
}
if (cpi->ref_frame_flags & av1_ref_frame_flag_list[ALTREF_FRAME]) {
const YV12_BUFFER_CONFIG *const ref =
diff --git a/av1/encoder/encoder_utils.c b/av1/encoder/encoder_utils.c
index f9e446b..c35873d 100644
--- a/av1/encoder/encoder_utils.c
+++ b/av1/encoder/encoder_utils.c
@@ -706,6 +706,14 @@
if (ref_frame == ALTREF_FRAME && cpi->svc.skip_mvsearch_altref)
continue;
}
+ // For RTC with superres on: golden reference only needs to be scaled
+ // if it was refreshed in previous frame.
+ if (is_one_pass_rt_params(cpi) &&
+ cpi->oxcf.superres_cfg.enable_superres && ref_frame == GOLDEN_FRAME &&
+ cpi->rc.frame_num_last_gf_refresh <
+ (int)cm->current_frame.frame_number - 1) {
+ continue;
+ }
if (ref->y_crop_width != cm->width || ref->y_crop_height != cm->height) {
// Replace the reference buffer with a copy having a thicker border,
diff --git a/av1/encoder/encoder_utils.h b/av1/encoder/encoder_utils.h
index 196676e..10ff92d 100644
--- a/av1/encoder/encoder_utils.h
+++ b/av1/encoder/encoder_utils.h
@@ -1013,10 +1013,23 @@
}
static AOM_INLINE void release_scaled_references(AV1_COMP *cpi) {
- // TODO(isbs): only refresh the necessary frames, rather than all of them
+ // Scaled references should only need to be released under certain conditions:
+ // if the reference will be updated, or if the scaled reference has same
+ // resolution. For now only apply this to Golden for non-svc RTC mode.
+ AV1_COMMON *const cm = &cpi->common;
+ const bool refresh_golden = (cpi->refresh_frame.golden_frame) ? 1 : 0;
+ bool release_golden = true;
for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
RefCntBuffer *const buf = cpi->scaled_ref_buf[i];
- if (buf != NULL) {
+ const int golden_ref = (i == GOLDEN_FRAME - 1);
+ if (golden_ref && is_one_pass_rt_params(cpi) && !cpi->ppi->use_svc &&
+ buf != NULL) {
+ const RefCntBuffer *const ref = get_ref_frame_buf(cm, GOLDEN_FRAME);
+ const bool same_resoln = buf->buf.y_crop_width == ref->buf.y_crop_width &&
+ buf->buf.y_crop_height == ref->buf.y_crop_height;
+ release_golden = refresh_golden || same_resoln;
+ }
+ if (buf != NULL && (!golden_ref || (golden_ref && release_golden))) {
--buf->ref_count;
cpi->scaled_ref_buf[i] = NULL;
}
diff --git a/av1/encoder/nonrd_opt.h b/av1/encoder/nonrd_opt.h
index a53578e..4fd88a7 100644
--- a/av1/encoder/nonrd_opt.h
+++ b/av1/encoder/nonrd_opt.h
@@ -435,6 +435,7 @@
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
MB_MODE_INFO_EXT *const mbmi_ext = &x->mbmi_ext;
+ int base_is_key_frame = 0;
const YV12_BUFFER_CONFIG *ref = get_ref_frame_yv12_buf(cm, ref_frame);
const bool ref_is_scaled =
ref->y_crop_height != cm->height || ref->y_crop_width != cm->width;
@@ -450,6 +451,13 @@
// TODO(kyslov) this needs various further optimizations. to be continued..
assert(yv12 != NULL);
if (yv12 != NULL) {
+ if (cpi->ppi->use_svc) {
+ SVC *const svc = &cpi->svc;
+ const int layer =
+ LAYER_IDS_TO_IDX(svc->spatial_layer_id, svc->temporal_layer_id,
+ svc->number_temporal_layers);
+ base_is_key_frame = svc->layer_context[layer].is_key_frame;
+ }
struct scale_factors *const sf =
scaled_ref ? NULL : get_ref_scale_factors(cm, ref_frame);
av1_setup_pred_block(xd, yv12_mb[ref_frame], yv12, sf, sf, num_planes);
@@ -463,9 +471,11 @@
cm->features.allow_high_precision_mv, mbmi_ext, ref_frame,
&frame_mv[NEARESTMV][ref_frame], &frame_mv[NEARMV][ref_frame], 0);
frame_mv[GLOBALMV][ref_frame] = mbmi_ext->global_mvs[ref_frame];
- // Early exit for non-LAST frame if force_skip_low_temp_var is set.
- if (!ref_is_scaled && bsize >= BLOCK_8X8 && !skip_pred_mv &&
- !(force_skip_low_temp_var && ref_frame != LAST_FRAME)) {
+ // Early exit for non-LAST frame if force_skip_low_temp_var or
+ // ref_is_scaled is set.
+ if (bsize >= BLOCK_8X8 && !skip_pred_mv && !base_is_key_frame &&
+ !((ref_is_scaled || force_skip_low_temp_var) &&
+ ref_frame != LAST_FRAME)) {
av1_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride, ref_frame,
bsize);
}
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 5dd54a5..1836869 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1762,7 +1762,7 @@
x->nonrd_prune_ref_frame_search > 2 &&
x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_U)] == 0 &&
x->color_sensitivity_sb_g[COLOR_SENS_IDX(AOM_PLANE_V)] == 0) {
- int thr = (cm->width * cm->height >= 640 * 360) ? 100 : 150;
+ int thr = (cm->width * cm->height > RESOLUTION_288P) ? 100 : 150;
int pred = x->pred_mv_sad[LAST_FRAME] >>
(b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize]);
if (pred > thr) use_golden_ref_frame = 1;
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 5505db2..d545600 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -1347,6 +1347,8 @@
AV1_COMMON *const cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
const int num_planes = av1_num_planes(cm);
+ bool scaled_ref_golden = false;
+ bool scaled_ref_alt = false;
BLOCK_SIZE bsize = is_small_sb ? BLOCK_64X64 : BLOCK_128X128;
MB_MODE_INFO *mi = xd->mi[0];
const YV12_BUFFER_CONFIG *yv12 =
@@ -1364,21 +1366,22 @@
cpi->sf.rt_sf.use_nonrd_altref_frame ||
(cpi->sf.rt_sf.use_comp_ref_nonrd &&
cpi->sf.rt_sf.ref_frame_comp_nonrd[2] == 1);
- // On a resized frame (reference has different scale) only use
- // LAST as reference for partitioning for now.
- if (scaled_ref_last) {
- use_golden_ref = 0;
- use_alt_ref = 0;
- }
// For 1 spatial layer: GOLDEN is another temporal reference.
// Check if it should be used as reference for partitioning.
if (cpi->svc.number_spatial_layers == 1 && use_golden_ref &&
(x->content_state_sb.source_sad_nonrd != kZeroSad || !use_last_ref)) {
yv12_g = get_ref_frame_yv12_buf(cm, GOLDEN_FRAME);
+ if (yv12_g && (yv12_g->y_crop_height != cm->height ||
+ yv12_g->y_crop_width != cm->width)) {
+ yv12_g = av1_get_scaled_ref_frame(cpi, GOLDEN_FRAME);
+ scaled_ref_golden = true;
+ }
if (yv12_g && yv12_g != yv12) {
- av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
- get_ref_scale_factors(cm, GOLDEN_FRAME), num_planes);
+ av1_setup_pre_planes(
+ xd, 0, yv12_g, mi_row, mi_col,
+ scaled_ref_golden ? NULL : get_ref_scale_factors(cm, GOLDEN_FRAME),
+ num_planes);
*y_sad_g = cpi->ppi->fn_ptr[bsize].sdf(
x->plane[AOM_PLANE_Y].src.buf, x->plane[AOM_PLANE_Y].src.stride,
xd->plane[AOM_PLANE_Y].pre[0].buf,
@@ -1392,9 +1395,16 @@
(cpi->ref_frame_flags & AOM_ALT_FLAG) &&
(x->content_state_sb.source_sad_nonrd != kZeroSad || !use_last_ref)) {
yv12_alt = get_ref_frame_yv12_buf(cm, ALTREF_FRAME);
+ if (yv12_alt && (yv12_alt->y_crop_height != cm->height ||
+ yv12_alt->y_crop_width != cm->width)) {
+ yv12_alt = av1_get_scaled_ref_frame(cpi, ALTREF_FRAME);
+ scaled_ref_alt = true;
+ }
if (yv12_alt && yv12_alt != yv12) {
- av1_setup_pre_planes(xd, 0, yv12_alt, mi_row, mi_col,
- get_ref_scale_factors(cm, ALTREF_FRAME), num_planes);
+ av1_setup_pre_planes(
+ xd, 0, yv12_alt, mi_row, mi_col,
+ scaled_ref_alt ? NULL : get_ref_scale_factors(cm, ALTREF_FRAME),
+ num_planes);
*y_sad_alt = cpi->ppi->fn_ptr[bsize].sdf(
x->plane[AOM_PLANE_Y].src.buf, x->plane[AOM_PLANE_Y].src.stride,
xd->plane[AOM_PLANE_Y].pre[0].buf,
