rtc: Allow for compound modes for nonrd_pickmode
For nonrd mode (speed >= 7): add LAST_GOLDEN compound
average mode for 0_0/NEAREST_NEAREST/NEAR_NEAR,
avg. bdrate gain rtc set: ~5.3/5/3.5% for speed 7/8/9.
avg. bdrate gain rtc_derf: ~0.5% for speed 7.
Speed slowdown of ~10-15% on 360p/480p clips.
Keep the speed feature (use_comp_ref_nonrd) off for now
(it's already off as default).
Change-Id: I4cb6687427535eb05b0a2e0d2e5192b4ce41ce90
diff --git a/av1/encoder/aq_cyclicrefresh.c b/av1/encoder/aq_cyclicrefresh.c
index 40b8c25..162cf8a 100644
--- a/av1/encoder/aq_cyclicrefresh.c
+++ b/av1/encoder/aq_cyclicrefresh.c
@@ -58,12 +58,14 @@
const MB_MODE_INFO *mbmi, int64_t rate,
int64_t dist, int bsize) {
MV mv = mbmi->mv[0].as_mv;
- // Reject the block for lower-qp coding if projected distortion
- // is above the threshold, and any of the following is true:
+ int is_compound = has_second_ref(mbmi);
+ // Reject the block for lower-qp coding for non-compound mode if
+ // projected distortion is above the threshold, and any of the following
+ // is true:
// 1) mode uses large mv
// 2) mode is an intra-mode
// Otherwise accept for refresh.
- if (dist > cr->thresh_dist_sb &&
+ if (!is_compound && dist > cr->thresh_dist_sb &&
(mv.row > cr->motion_thresh || mv.row < -cr->motion_thresh ||
mv.col > cr->motion_thresh || mv.col < -cr->motion_thresh ||
!is_inter_block(mbmi)))
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 136a5db..1e63fee 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1736,7 +1736,8 @@
(void)num_planes;
#endif
- if (cpi->sf.hl_sf.frame_parameter_update) {
+ if (cpi->sf.hl_sf.frame_parameter_update ||
+ cpi->sf.rt_sf.use_comp_ref_nonrd) {
RD_COUNTS *const rdc = &cpi->td.rd_counts;
if (frame_is_intra_only(cm))
@@ -1780,6 +1781,10 @@
features->tx_mode = TX_MODE_LARGEST;
}
} else {
+ // This is needed if real-time speed setting is changed on the fly
+ // from one using compound prediction to one using single reference.
+ if (current_frame->reference_mode == REFERENCE_MODE_SELECT)
+ current_frame->reference_mode = SINGLE_REFERENCE;
encode_frame_internal(cpi);
}
}
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 088135a..851a8ff 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -48,6 +48,7 @@
PREDICTION_MODE best_mode;
TX_SIZE best_tx_size;
MV_REFERENCE_FRAME best_ref_frame;
+ MV_REFERENCE_FRAME best_second_ref_frame;
uint8_t best_mode_skip_txfm;
uint8_t best_mode_initial_skip_flag;
int_interpfilters best_pred_filter;
@@ -119,6 +120,7 @@
static INLINE void init_best_pickmode(BEST_PICKMODE *bp) {
bp->best_mode = NEARESTMV;
bp->best_ref_frame = LAST_FRAME;
+ bp->best_second_ref_frame = NONE_FRAME;
bp->best_tx_size = TX_8X8;
bp->best_pred_filter = av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
bp->best_mode_skip_txfm = 0;
@@ -1977,6 +1979,7 @@
best_pickmode->best_mode = this_mode;
best_pickmode->best_tx_size = mi->tx_size;
best_pickmode->best_ref_frame = INTRA_FRAME;
+ best_pickmode->best_second_ref_frame = NONE;
mi->uv_mode = this_mode;
mi->mv[0].as_int = INVALID_MV;
mi->mv[1].as_int = INVALID_MV;
@@ -2104,6 +2107,38 @@
}
}
+static void set_compound_mode(MACROBLOCK *x, int idx, int num_inter_modes,
+ int ref_frame, int ref_frame2, int ref_mv_idx,
+ int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES],
+ PREDICTION_MODE *this_mode) {
+ MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *const mi = xd->mi[0];
+ *this_mode = GLOBAL_GLOBALMV;
+ mi->ref_frame[0] = ref_frame;
+ mi->ref_frame[1] = ref_frame2;
+ mi->compound_idx = 1;
+ mi->comp_group_idx = 0;
+ mi->interinter_comp.type = COMPOUND_AVERAGE;
+ const MV_REFERENCE_FRAME ref_frame_comp =
+ REF_FRAMES + FWD_REFS * BWD_REFS + LAST_GOLDEN_FRAMES;
+ if (idx == num_inter_modes) {
+ frame_mv[*this_mode][ref_frame].as_int = 0;
+ frame_mv[*this_mode][ref_frame2].as_int = 0;
+ } else if (idx == num_inter_modes + 1) {
+ *this_mode = NEAREST_NEARESTMV;
+ frame_mv[*this_mode][ref_frame].as_int =
+ xd->ref_mv_stack[ref_frame_comp][0].this_mv.as_int;
+ frame_mv[*this_mode][ref_frame2].as_int =
+ xd->ref_mv_stack[ref_frame_comp][0].comp_mv.as_int;
+ } else if (idx == num_inter_modes + 2) {
+ *this_mode = NEAR_NEARMV;
+ frame_mv[*this_mode][ref_frame].as_int =
+ xd->ref_mv_stack[ref_frame_comp][ref_mv_idx].this_mv.as_int;
+ frame_mv[*this_mode][ref_frame2].as_int =
+ xd->ref_mv_stack[ref_frame_comp][ref_mv_idx].comp_mv.as_int;
+ }
+}
+
void av1_nonrd_pick_inter_mode_sb(AV1_COMP *cpi, TileDataEnc *tile_data,
MACROBLOCK *x, RD_STATS *rd_cost,
BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx) {
@@ -2117,7 +2152,7 @@
#if COLLECT_PICK_MODE_STAT
static mode_search_stat ms_stat;
#endif
- MV_REFERENCE_FRAME ref_frame;
+ MV_REFERENCE_FRAME ref_frame, ref_frame2;
int_mv frame_mv[MB_MODE_COUNT][REF_FRAMES];
uint8_t mode_checked[MB_MODE_COUNT][REF_FRAMES];
struct buf_2d yv12_mb[REF_FRAMES][MAX_MB_PLANE];
@@ -2165,6 +2200,9 @@
int svc_mv_row = 0;
int force_mv_inter_layer = 0;
int use_modeled_non_rd_cost = 0;
+ int comp_pred = 0;
+ int num_comp_modes = 0;
+ int ref_mv_idx = 0;
#if CONFIG_AV1_TEMPORAL_DENOISING
const int denoise_recheck_zeromv = 1;
AV1_PICKMODE_CTX_DEN ctx_den;
@@ -2236,6 +2274,10 @@
get_ref_frame_use_mask(cpi, x, mi, mi_row, mi_col, bsize, gf_temporal_ref,
use_ref_frame_mask, &force_skip_low_temp_var);
+ // Test compound modes: LAST_GOLDEN for 0/NEARESST/NEAR.
+ if (cpi->sf.rt_sf.use_comp_ref_nonrd && is_comp_ref_allowed(bsize))
+ num_comp_modes = 3;
+
for (MV_REFERENCE_FRAME ref_frame_iter = LAST_FRAME;
ref_frame_iter <= ALTREF_FRAME; ++ref_frame_iter) {
if (use_ref_frame_mask[ref_frame_iter]) {
@@ -2243,6 +2285,17 @@
bsize, force_skip_low_temp_var);
}
}
+ if (num_comp_modes > 0) {
+ MB_MODE_INFO *const mbmi = xd->mi[0];
+ MB_MODE_INFO_EXT *const mbmi_ext = &x->mbmi_ext;
+ const MV_REFERENCE_FRAME ref_frame_comp =
+ REF_FRAMES + FWD_REFS * BWD_REFS + LAST_GOLDEN_FRAMES;
+ av1_find_mv_refs(cm, xd, mbmi, ref_frame_comp, mbmi_ext->ref_mv_count,
+ xd->ref_mv_stack, xd->weight, NULL, mbmi_ext->global_mvs,
+ mbmi_ext->mode_context);
+ av1_copy_usable_ref_mv_stack_and_weight(xd, mbmi_ext, ref_frame_comp);
+ ref_mv_idx = mbmi->ref_mv_idx + 1;
+ }
thresh_sad_pred = ((int64_t)x->pred_mv_sad[LAST_FRAME]) << 1;
// Increase threshold for less agressive pruning.
@@ -2282,20 +2335,35 @@
tx_mode_to_biggest_tx_size[txfm_params->tx_mode_search_type]),
TX_16X16);
- for (int idx = 0; idx < num_inter_modes; ++idx) {
+ for (int idx = 0; idx < num_inter_modes + num_comp_modes; ++idx) {
const struct segmentation *const seg = &cm->seg;
int rate_mv = 0;
int is_skippable;
int this_early_term = 0;
int skip_this_mv = 0;
+ comp_pred = 0;
PREDICTION_MODE this_mode;
MB_MODE_INFO_EXT *const mbmi_ext = &x->mbmi_ext;
RD_STATS nonskip_rdc;
av1_invalid_rd_stats(&nonskip_rdc);
- this_mode = ref_mode_set[idx].pred_mode;
- ref_frame = ref_mode_set[idx].ref_frame;
+ if (idx >= num_inter_modes) {
+ ref_frame = LAST_FRAME;
+ ref_frame2 = GOLDEN_FRAME;
+ if (!use_ref_frame_mask[ref_frame2]) continue;
+ set_compound_mode(x, idx, num_inter_modes, ref_frame, ref_frame2,
+ ref_mv_idx, frame_mv, &this_mode);
+ if (this_mode != GLOBAL_GLOBALMV &&
+ frame_mv[this_mode][ref_frame].as_int == 0 &&
+ frame_mv[this_mode][ref_frame2].as_int == 0)
+ continue;
+ comp_pred = 1;
+ } else {
+ this_mode = ref_mode_set[idx].pred_mode;
+ ref_frame = ref_mode_set[idx].ref_frame;
+ ref_frame2 = NONE_FRAME;
+ }
#if COLLECT_PICK_MODE_STAT
aom_usec_timer_start(&ms_stat.timer1);
@@ -2303,6 +2371,7 @@
#endif
mi->mode = this_mode;
mi->ref_frame[0] = ref_frame;
+ mi->ref_frame[1] = ref_frame2;
if (!use_ref_frame_mask[ref_frame]) continue;
@@ -2354,21 +2423,24 @@
x->pred_mv1_sad[ref_frame] > (x->pred_mv0_sad[ref_frame] << 1))
continue;
- if (skip_mode_by_threshold(
- this_mode, ref_frame, frame_mv[this_mode][ref_frame],
- cpi->rc.frames_since_golden, rd_threshes, rd_thresh_freq_fact,
- best_rdc.rdcost, best_pickmode.best_mode_skip_txfm,
- (cpi->sf.rt_sf.nonrd_agressive_skip ? 1 : 0)))
- continue;
+ if (!comp_pred) {
+ if (skip_mode_by_threshold(
+ this_mode, ref_frame, frame_mv[this_mode][ref_frame],
+ cpi->rc.frames_since_golden, rd_threshes, rd_thresh_freq_fact,
+ best_rdc.rdcost, best_pickmode.best_mode_skip_txfm,
+ (cpi->sf.rt_sf.nonrd_agressive_skip ? 1 : 0)))
+ continue;
+ }
// Select prediction reference frames.
for (int i = 0; i < MAX_MB_PLANE; i++) {
xd->plane[i].pre[0] = yv12_mb[ref_frame][i];
+ if (comp_pred) xd->plane[i].pre[1] = yv12_mb[ref_frame2][i];
}
mi->ref_frame[0] = ref_frame;
- mi->ref_frame[1] = NONE_FRAME;
- set_ref_ptrs(cm, xd, ref_frame, NONE_FRAME);
+ mi->ref_frame[1] = ref_frame2;
+ set_ref_ptrs(cm, xd, ref_frame, ref_frame2);
if (this_mode == NEWMV && !force_mv_inter_layer) {
if (search_new_mv(cpi, x, frame_mv, ref_frame, gf_temporal_ref, bsize,
@@ -2387,11 +2459,13 @@
}
}
- if (skip_this_mv) continue;
+ if (skip_this_mv && !comp_pred) continue;
mi->mode = this_mode;
mi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int;
mi->mv[1].as_int = 0;
+ if (comp_pred) mi->mv[1].as_int = frame_mv[this_mode][ref_frame2].as_int;
+
if (reuse_inter_pred) {
if (!this_mode_pred) {
this_mode_pred = &tmp[3];
@@ -2420,7 +2494,7 @@
set_color_sensitivity(cpi, x, xd, bsize, y_sad, x->source_variance);
}
- if (enable_filter_search && !force_mv_inter_layer &&
+ if (enable_filter_search && !force_mv_inter_layer && !comp_pred &&
((mi->mv[0].as_mv.row & 0x07) || (mi->mv[0].as_mv.col & 0x07)) &&
(ref_frame == LAST_FRAME || !x->nonrd_prune_ref_frame_search)) {
search_filter_ref(cpi, x, &this_rdc, mi_row, mi_col, tmp, bsize,
@@ -2434,7 +2508,8 @@
if (force_mv_inter_layer)
mi->interp_filters = av1_broadcast_interp_filter(EIGHTTAP_REGULAR);
- av1_enc_build_inter_predictor_y(xd, mi_row, mi_col);
+ av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, bsize, 0, 0);
+
if (use_model_yrd_large) {
model_skip_for_sb_y_large(cpi, bsize, mi_row, mi_col, x, xd, &this_rdc,
&this_early_term, use_modeled_non_rd_cost);
@@ -2516,7 +2591,7 @@
this_rdc.rate += ref_costs_single[ref_frame];
this_rdc.rdcost = RDCOST(x->rdmult, this_rdc.rate, this_rdc.dist);
- if (cpi->oxcf.rc_cfg.mode == AOM_CBR) {
+ if (cpi->oxcf.rc_cfg.mode == AOM_CBR && !comp_pred) {
newmv_diff_bias(xd, this_mode, &this_rdc, bsize,
frame_mv[this_mode][ref_frame].as_mv.row,
frame_mv[this_mode][ref_frame].as_mv.col, cpi->speed,
@@ -2547,6 +2622,7 @@
best_pickmode.best_pred_filter = mi->interp_filters;
best_pickmode.best_tx_size = mi->tx_size;
best_pickmode.best_ref_frame = ref_frame;
+ best_pickmode.best_second_ref_frame = ref_frame2;
best_pickmode.best_mode_skip_txfm = this_rdc.skip_txfm;
best_pickmode.best_mode_initial_skip_flag =
(nonskip_rdc.rate == INT_MAX && this_rdc.skip_txfm);
@@ -2571,7 +2647,13 @@
mi->ref_frame[0] = best_pickmode.best_ref_frame;
mi->mv[0].as_int =
frame_mv[best_pickmode.best_mode][best_pickmode.best_ref_frame].as_int;
-
+ mi->mv[1].as_int = 0;
+ if (best_pickmode.best_second_ref_frame > INTRA_FRAME) {
+ mi->ref_frame[1] = best_pickmode.best_second_ref_frame;
+ mi->mv[1].as_int =
+ frame_mv[best_pickmode.best_mode][best_pickmode.best_second_ref_frame]
+ .as_int;
+ }
// Perform intra prediction search, if the best SAD is above a certain
// threshold.
mi->angle_delta[PLANE_TYPE_Y] = 0;
@@ -2586,7 +2668,13 @@
pd->dst = orig_dst;
mi->mode = best_pickmode.best_mode;
mi->ref_frame[0] = best_pickmode.best_ref_frame;
+ mi->ref_frame[1] = best_pickmode.best_second_ref_frame;
txfm_info->skip_txfm = best_rdc.skip_txfm;
+ if (has_second_ref(mi)) {
+ mi->comp_group_idx = 0;
+ mi->compound_idx = 1;
+ mi->interinter_comp.type = COMPOUND_AVERAGE;
+ }
if (!is_inter_block(mi)) {
mi->interp_filters = av1_broadcast_interp_filter(SWITCHABLE_FILTERS);
@@ -2618,7 +2706,7 @@
}
#endif
- if (cpi->sf.inter_sf.adaptive_rd_thresh) {
+ if (cpi->sf.inter_sf.adaptive_rd_thresh && !has_second_ref(mi)) {
THR_MODES best_mode_idx =
mode_idx[best_pickmode.best_ref_frame][mode_offset(mi->mode)];
if (best_pickmode.best_ref_frame == INTRA_FRAME) {
diff --git a/av1/encoder/partition_search.c b/av1/encoder/partition_search.c
index fa09e7e..d3135ca 100644
--- a/av1/encoder/partition_search.c
+++ b/av1/encoder/partition_search.c
@@ -1841,6 +1841,7 @@
int mi_col, RUN_TYPE dry_run, BLOCK_SIZE bsize,
PARTITION_TYPE partition,
PICK_MODE_CONTEXT *const ctx, int *rate) {
+ const AV1_COMMON *const cm = &cpi->common;
TileInfo *const tile = &tile_data->tile_info;
MACROBLOCK *const x = &td->mb;
MACROBLOCKD *xd = &x->e_mbd;
@@ -1849,8 +1850,6 @@
setup_block_rdmult(cpi, x, mi_row, mi_col, bsize, NO_AQ, NULL);
MB_MODE_INFO *mbmi = xd->mi[0];
mbmi->partition = partition;
- // Nonrd pickmode does not currently support second/combined reference.
- assert(!has_second_ref(mbmi));
av1_update_state(cpi, td, ctx, mi_row, mi_col, bsize, dry_run);
const int subsampling_x = cpi->common.seq_params->subsampling_x;
const int subsampling_y = cpi->common.seq_params->subsampling_y;
@@ -1866,6 +1865,41 @@
encode_superblock(cpi, tile_data, td, tp, dry_run, bsize, rate);
if (!dry_run) {
update_cb_offsets(x, bsize, subsampling_x, subsampling_y);
+ if (has_second_ref(mbmi)) {
+ if (mbmi->compound_idx == 0 ||
+ mbmi->interinter_comp.type == COMPOUND_AVERAGE)
+ mbmi->comp_group_idx = 0;
+ else
+ mbmi->comp_group_idx = 1;
+ mbmi->compound_idx = 1;
+ }
+ RD_COUNTS *const rdc = &td->rd_counts;
+ if (mbmi->skip_mode) {
+ assert(!frame_is_intra_only(cm));
+ rdc->skip_mode_used_flag = 1;
+ if (cm->current_frame.reference_mode == REFERENCE_MODE_SELECT &&
+ has_second_ref(mbmi)) {
+ rdc->compound_ref_used_flag = 1;
+ }
+ set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]);
+ } else {
+ const int seg_ref_active =
+ segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_REF_FRAME);
+ if (!seg_ref_active) {
+ // If the segment reference feature is enabled we have only a single
+ // reference frame allowed for the segment so exclude it from
+ // the reference frame counts used to work out probabilities.
+ if (is_inter_block(mbmi)) {
+ av1_collect_neighbors_ref_counts(xd);
+ if (cm->current_frame.reference_mode == REFERENCE_MODE_SELECT &&
+ has_second_ref(mbmi)) {
+ // This flag is also updated for 4x4 blocks
+ rdc->compound_ref_used_flag = 1;
+ }
+ set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]);
+ }
+ }
+ }
if (tile_data->allow_update_cdf) update_stats(&cpi->common, td);
}
if (cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ && mbmi->skip_txfm)
