Support ext-skip for both low delay and high delay
For both low delay and high delay scenarios, the reference pair in
skip mode are specified as the closest fwd ref, together with the
closest bwd ref if there is any bwd ref, otherwise with the two
closest fwd refs.
Skip mode by default uses COMPOUND_AVERAGE. When all the reference
frames are on the same side, temporal-distance weighted compound is
considered, and a compound index is signaled to indicate whether
distance-weighted compound or compound-average is usd.
Whether to use distance-weighted compound for skip mode is still
under experimenting, hence a flag is temporarily added:
SKIP_MODE_WITH_JNT_COMP.
Following experimental results are obtained over 30 frames, using the
setup of --disable-ext-partition --disable-ext-partition-types
--disable-txmg --enable-jnt-comp --enable-mfmv --enable-ext-skip:
(1) High Latency:
For Google test sets (lowres/midres), the BDRate coding gain is ~0.2%;
For AWCY, the coding gain is ~0.1%.
(2) Low Latency:
No gain has been observed over Google sets and ~0.1% gain is obtained
only when temporal-distance weighted prediction is used.
Change-Id: I8c433357adebed0126ebfdd5c4d51aa71e64be57
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 361eaf5..50281d3 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -47,6 +47,11 @@
// chroma.
#define DISABLE_VARTX_FOR_CHROMA 1
+#if CONFIG_EXT_SKIP && CONFIG_JNT_COMP
+// NOTE: To explore whether weighted-compound prediction to be considered.
+#define SKIP_MODE_WITH_JNT_COMP 0
+#endif // CONFIG_EXT_SKIP && CONFIG_JNT_COMP
+
// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
typedef enum {
#if COMPOUND_SEGMENT_TYPE == 0
@@ -706,6 +711,10 @@
#if CONFIG_JNT_COMP
JNT_COMP_PARAMS jcp_param;
#endif
+
+#if CONFIG_EXT_SKIP
+ int all_one_sided_refs;
+#endif // CONFIG_EXT_SKIP
} MACROBLOCKD;
static INLINE int get_bitdepth_data_path_index(const MACROBLOCKD *xd) {
diff --git a/av1/common/mvref_common.c b/av1/common/mvref_common.c
index dfd43762b..ab2d141 100644
--- a/av1/common/mvref_common.c
+++ b/av1/common/mvref_common.c
@@ -2054,9 +2054,9 @@
}
// Flag is set when and only when both forward and backward references
- // are available and their distance is no greater than 3, i.e. as
- // opposed to the current frame position, the reference distance pair are
- // either: (1, 1), (1, 2), or (2, 1).
+ // are available and the difference between their temporal distance to
+ // the current frame is no greater than 1, i.e. they may have the same
+ // temporal distance to the current frame, or the distances are off by 1.
if (ref_idx[0] != INVALID_IDX && ref_idx[1] != INVALID_IDX) {
const int cur_to_fwd = cm->frame_offset - ref_frame_offset[0];
const int cur_to_bwd = ref_frame_offset[1] - cm->frame_offset;
@@ -2070,6 +2070,38 @@
}
}
+ // NOTE: Low delay scenario
+ if (av1_refs_are_one_sided(cm)) {
+ assert(ref_idx[1] == INVALID_IDX);
+ assert(ref_idx[0] != INVALID_IDX);
+ // Identify the second closest forward reference frame.
+ ref_frame_offset[1] = -1;
+ cm->is_skip_mode_allowed = 0;
+ for (int i = 0; i < INTER_REFS_PER_FRAME; ++i) {
+ const int buf_idx = cm->frame_refs[i].idx;
+ if (buf_idx == INVALID_IDX) continue;
+
+ const int ref_offset = frame_bufs[buf_idx].cur_frame_offset;
+ if (ref_offset < ref_frame_offset[0] &&
+ ref_offset > ref_frame_offset[1]) {
+ // Second closest forward reference
+ ref_frame_offset[1] = ref_offset;
+ ref_idx[1] = i;
+ ref_buf_idx[1] = buf_idx;
+ }
+ }
+ if (ref_frame_offset[1] >= 0) {
+ const int cur_to_fwd0 = cur_frame_offset - ref_frame_offset[0];
+ const int cur_to_fwd1 = cur_frame_offset - ref_frame_offset[1];
+ assert(cur_to_fwd1 > cur_to_fwd0);
+ if ((cur_to_fwd1 - cur_to_fwd0) <= 1) {
+ cm->is_skip_mode_allowed = 1;
+ cm->ref_frame_idx_0 = AOMMIN(ref_idx[0], ref_idx[1]);
+ cm->ref_frame_idx_1 = AOMMAX(ref_idx[0], ref_idx[1]);
+ }
+ }
+ }
+
// Set up the temporal mv candidates for skip mode.
cm->tpl_frame_ref0_idx = INVALID_IDX;
if (cm->is_skip_mode_allowed && cm->use_ref_frame_mvs) {
diff --git a/av1/common/mvref_common.h b/av1/common/mvref_common.h
index 67bfca6..e28eaea 100644
--- a/av1/common/mvref_common.h
+++ b/av1/common/mvref_common.h
@@ -402,6 +402,24 @@
}
#if CONFIG_FRAME_MARKER
+static INLINE int av1_refs_are_one_sided(const AV1_COMMON *cm) {
+ assert(!frame_is_intra_only(cm));
+
+ int one_sided_refs = 1;
+ for (int ref = 0; ref < INTER_REFS_PER_FRAME; ++ref) {
+ const int buf_idx = cm->frame_refs[ref].idx;
+ if (buf_idx == INVALID_IDX) continue;
+
+ const int ref_offset =
+ cm->buffer_pool->frame_bufs[buf_idx].cur_frame_offset;
+ if (ref_offset > (int)cm->frame_offset) {
+ one_sided_refs = 0; // bwd reference
+ break;
+ }
+ }
+ return one_sided_refs;
+}
+
void av1_setup_frame_buf_refs(AV1_COMMON *cm);
#if CONFIG_FRAME_SIGN_BIAS
void av1_setup_frame_sign_bias(AV1_COMMON *cm);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 0514f45..184b361 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -2996,6 +2996,7 @@
#if CONFIG_EXT_SKIP
av1_setup_skip_mode_allowed(cm);
cm->skip_mode_flag = cm->is_skip_mode_allowed ? aom_rb_read_bit(rb) : 0;
+ xd->all_one_sided_refs = cm->skip_mode_flag ? av1_refs_are_one_sided(cm) : 0;
#if 0
printf(
"DECODER: Frame=%d, frame_offset=%d, show_frame=%d, "
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 9b708af..df9d8de 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -2210,6 +2210,24 @@
mbmi->compound_idx = 1;
mbmi->interinter_compound_type = COMPOUND_AVERAGE;
+#if CONFIG_EXT_SKIP
+#if CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
+ if (mbmi->skip_mode) {
+ const int cur_offset = (int)cm->frame_offset;
+ const int cur_to_fwd = cur_offset - cm->ref_frame_idx_0;
+ const int cur_to_bwd = abs(cm->ref_frame_idx_1 - cur_offset);
+ if (cur_to_fwd != cur_to_bwd && xd->all_one_sided_refs) {
+ const int comp_index_ctx = get_comp_index_context(cm, xd);
+ mbmi->compound_idx = aom_read_symbol(
+ r, ec_ctx->compound_index_cdf[comp_index_ctx], 2, ACCT_STR);
+
+ if (xd->counts)
+ ++xd->counts->compound_index[comp_index_ctx][mbmi->compound_idx];
+ }
+ }
+#endif // CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
+#endif // CONFIG_EXT_SKIP
+
if (has_second_ref(mbmi)
#if CONFIG_EXT_SKIP
&& !mbmi->skip_mode
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 8f5eec5..99ee4f6 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1410,7 +1410,19 @@
}
#if CONFIG_EXT_SKIP
- if (mbmi->skip_mode) return;
+ if (mbmi->skip_mode) {
+#if CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
+ const int cur_offset = (int)cm->frame_offset;
+ const int cur_to_fwd = cur_offset - cm->ref_frame_idx_0;
+ const int cur_to_bwd = abs(cm->ref_frame_idx_1 - cur_offset);
+ if (cur_to_fwd != cur_to_bwd && xd->all_one_sided_refs) {
+ const int comp_index_ctx = get_comp_index_context(cm, xd);
+ aom_write_symbol(w, mbmi->compound_idx,
+ ec_ctx->compound_index_cdf[comp_index_ctx], 2);
+ }
+#endif // CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
+ return;
+ }
#endif // CONFIG_EXT_SKIP
if (!is_inter) {
@@ -4305,7 +4317,11 @@
arf_offset = AOMMIN((MAX_GF_INTERVAL - 1), arf_offset + brf_offset);
aom_wb_write_literal(wb, arf_offset, FRAME_OFFSET_BITS);
}
-#endif
+
+#if CONFIG_EXT_SKIP
+ if (cm->is_skip_mode_allowed) aom_wb_write_bit(wb, cm->skip_mode_flag);
+#endif // CONFIG_EXT_SKIP
+#endif // CONFIG_FRAME_MARKER
#if CONFIG_REFERENCE_BUFFER
if (cm->seq_params.frame_id_numbers_present_flag) {
diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 994642f..5126eb9 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -254,6 +254,9 @@
int64_t skip_mode_dist;
MV_REFERENCE_FRAME skip_mode_ref_frame[2];
int_mv skip_mode_mv[2];
+#if CONFIG_JNT_COMP
+ int compound_idx;
+#endif // CONFIG_JNT_COMP
int skip_mode_index_candidate;
int skip_mode_index;
#endif // CONFIG_EXT_SKIP
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 45b76c6..fae1fb6 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -872,6 +872,8 @@
if (!frame_is_intra_only(cm)) {
RD_COUNTS *rdc = &td->rd_counts;
+ FRAME_COUNTS *const counts = td->counts;
+
#if CONFIG_EXT_SKIP
if (mbmi->skip_mode) {
rdc->skip_mode_used_flag = 1;
@@ -879,12 +881,24 @@
assert(has_second_ref(mbmi));
rdc->compound_ref_used_flag = 1;
}
+#if CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
+ const int cur_offset = (int)cm->frame_offset;
+ const int cur_to_fwd = cur_offset - cm->ref_frame_idx_0;
+ const int cur_to_bwd = abs(cm->ref_frame_idx_1 - cur_offset);
+ if (cur_to_fwd != cur_to_bwd && xd->all_one_sided_refs) {
+ const int comp_index_ctx = get_comp_index_context(cm, xd);
+ ++counts->compound_index[comp_index_ctx][mbmi->compound_idx];
+ if (allow_update_cdf)
+ update_cdf(fc->compound_index_cdf[comp_index_ctx], mbmi->compound_idx,
+ 2);
+ }
+#endif // CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
set_ref_ptrs(cm, xd, mbmi->ref_frame[0], mbmi->ref_frame[1]);
+
return;
}
#endif // CONFIG_EXT_SKIP
- FRAME_COUNTS *const counts = td->counts;
const int inter_block = is_inter_block(mbmi);
if (!seg_ref_active) {
@@ -3555,19 +3569,6 @@
}
#if CONFIG_FRAME_MARKER
-static int refs_are_one_sided(const AV1_COMMON *cm) {
- int one_sided_refs = 1;
- if (cm->cur_frame->lst_frame_offset > cm->frame_offset ||
- cm->cur_frame->lst2_frame_offset > cm->frame_offset ||
- cm->cur_frame->lst3_frame_offset > cm->frame_offset ||
- cm->cur_frame->gld_frame_offset > cm->frame_offset ||
- cm->cur_frame->bwd_frame_offset > cm->frame_offset ||
- cm->cur_frame->alt2_frame_offset > cm->frame_offset ||
- cm->cur_frame->alt_frame_offset > cm->frame_offset)
- one_sided_refs = 0;
- return one_sided_refs;
-}
-
// Enforce the number of references for each arbitrary frame limited to
// (INTER_REFS_PER_FRAME - 1)
static void enforce_max_ref_frames(AV1_COMP *cpi) {
@@ -4014,9 +4015,19 @@
av1_setup_motion_field(cm);
#endif // CONFIG_MFMV
+#if CONFIG_FRAME_MARKER
+ cpi->all_one_sided_refs =
+ frame_is_intra_only(cm) ? 0 : av1_refs_are_one_sided(cm);
+#endif // CONFIG_FRAME_MARKER
+
#if CONFIG_EXT_SKIP
av1_setup_skip_mode_allowed(cm);
cm->skip_mode_flag = cm->is_skip_mode_allowed;
+ if (cm->skip_mode_flag && cpi->all_one_sided_refs &&
+ cpi->oxcf.lag_in_frames > 0) {
+ // High latency: Turn off skip mode if all refs are fwd.
+ cm->skip_mode_flag = 0;
+ }
if (cm->skip_mode_flag) {
if (cm->reference_mode == SINGLE_REFERENCE) {
cm->skip_mode_flag = 0;
@@ -4036,23 +4047,20 @@
cm->skip_mode_flag = 0;
}
}
+ xd->all_one_sided_refs = cm->skip_mode_flag ? cpi->all_one_sided_refs : 0;
#if 0
printf(
"\nENCODER: Frame=%d, frame_offset=%d, show_frame=%d, "
"show_existing_frame=%d, is_skip_mode_allowed=%d, "
"ref_frame_idx=(%d,%d), frame_reference_mode=%d, "
- "tpl_frame_ref0_idx=%d, skip_mode_flag=%d\n",
+ "tpl_frame_ref0_idx=%d, skip_mode_flag=%d, lag_in_frames=%d\n",
cm->current_video_frame, cm->frame_offset, cm->show_frame,
cm->show_existing_frame, cm->is_skip_mode_allowed, cm->ref_frame_idx_0,
cm->ref_frame_idx_1, cm->reference_mode, cm->tpl_frame_ref0_idx,
- cm->skip_mode_flag);
+ cm->skip_mode_flag, cpi->oxcf.lag_in_frames);
#endif // 0
#endif // CONFIG_EXT_SKIP
-#if CONFIG_FRAME_MARKER
- cpi->all_one_sided_refs = refs_are_one_sided(cm);
-#endif // CONFIG_FRAME_MARKER
-
{
struct aom_usec_timer emr_timer;
aom_usec_timer_start(&emr_timer);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index e8498fb..702fda2 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9191,7 +9191,7 @@
mbmi->interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1);
#if CONFIG_JNT_COMP
mbmi->comp_group_idx = 0;
- mbmi->compound_idx = 1;
+ mbmi->compound_idx = x->compound_idx;
#endif // CONFIG_JNT_COMP
mbmi->interinter_compound_type = COMPOUND_AVERAGE;
mbmi->motion_mode = SIMPLE_TRANSLATION;
@@ -10618,9 +10618,57 @@
if (cm->skip_mode_flag &&
!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) &&
is_comp_ref_allowed(bsize)) {
- // Obtain the rdcost for skip_mode.
- estimate_skip_mode_rdcost(cpi, tile_data, x, bsize, mi_row, mi_col,
- frame_mv, yv12_mb);
+// Obtain the rdcost for skip_mode.
+#if CONFIG_JNT_COMP
+ x->compound_idx = 1; // COMPOUND_AVERAGE
+#if SKIP_MODE_WITH_JNT_COMP
+ const int cur_offset = (int)cm->frame_offset;
+ const int cur_to_fwd = cur_offset - cm->ref_frame_idx_0;
+ const int cur_to_bwd = abs(cm->ref_frame_idx_1 - cur_offset);
+ if (cur_to_fwd != cur_to_bwd && xd->all_one_sided_refs) {
+ // Decide on the JNT_COMP mode.
+ int64_t best_skip_mode_rd = INT64_MAX;
+ int best_compound_idx = 0;
+ int best_skip_mode_rate = 0;
+ int64_t best_skip_mode_sse = 0, best_skip_mode_dist = 0;
+
+ for (int compound_idx = 0; compound_idx < 2; ++compound_idx) {
+ x->compound_idx = compound_idx;
+ estimate_skip_mode_rdcost(cpi, tile_data, x, bsize, mi_row, mi_col,
+ frame_mv, yv12_mb);
+
+ if (x->skip_mode_rdcost >= 0 && x->skip_mode_rdcost < INT64_MAX) {
+ // Update skip mode rdcost.
+ const int comp_index_ctx = get_comp_index_context(cm, xd);
+ x->skip_mode_rate += x->comp_idx_cost[comp_index_ctx][compound_idx];
+ x->skip_mode_rdcost =
+ RDCOST(x->rdmult, x->skip_mode_rate, x->skip_mode_dist);
+
+ if (x->skip_mode_rdcost < best_skip_mode_rd) {
+ best_skip_mode_rd = x->skip_mode_rdcost;
+ best_compound_idx = compound_idx;
+ best_skip_mode_rate = x->skip_mode_rate;
+ best_skip_mode_sse = x->skip_mode_sse;
+ best_skip_mode_dist = x->skip_mode_dist;
+ }
+ }
+ }
+
+ if (best_skip_mode_rd < INT64_MAX) {
+ x->compound_idx = best_compound_idx;
+ x->skip_mode_rdcost = best_skip_mode_rd;
+ x->skip_mode_rate = best_skip_mode_rate;
+ x->skip_mode_sse = best_skip_mode_sse;
+ x->skip_mode_dist = best_skip_mode_dist;
+ }
+ } else {
+#endif // SKIP_MODE_WITH_JNT_COMP
+#endif // CONFIG_JNT_COMP
+ estimate_skip_mode_rdcost(cpi, tile_data, x, bsize, mi_row, mi_col,
+ frame_mv, yv12_mb);
+#if CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
+ }
+#endif // CONFIG_JNT_COMP && SKIP_MODE_WITH_JNT_COMP
if (x->skip_mode_rdcost >= 0 && x->skip_mode_rdcost < INT64_MAX) {
// Update skip mode rdcost.
@@ -10668,9 +10716,15 @@
best_mbmode.uv_mode = UV_DC_PRED;
best_mbmode.palette_mode_info.palette_size[0] = 0;
best_mbmode.palette_mode_info.palette_size[1] = 0;
- best_mbmode.interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1);
+
+#if CONFIG_JNT_COMP
+ best_mbmode.comp_group_idx = 0;
+ best_mbmode.compound_idx = x->compound_idx;
+#endif // CONFIG_JNT_COMP
best_mbmode.interinter_compound_type = COMPOUND_AVERAGE;
best_mbmode.motion_mode = SIMPLE_TRANSLATION;
+
+ best_mbmode.interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1);
#if CONFIG_FILTER_INTRA
best_mbmode.filter_intra_mode_info.use_filter_intra_mode[0] = 0;
best_mbmode.filter_intra_mode_info.use_filter_intra_mode[1] = 0;
