Deprecate special rd loop for sub8x8 block size
Remove the special rate-distortion optimization loop for sub8x8
block size from vp9.
Change-Id: I62c6cf537a54769f26f2d4938ebed5fed2c84741
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 0c46d03..a288c02 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1323,7 +1323,6 @@
struct macroblockd_plane *const pd = xd->plane;
const AQ_MODE aq_mode = cpi->oxcf.aq_mode;
int i, orig_rdmult;
- const int unify_bsize = CONFIG_CB4X4;
aom_clear_system_state();
@@ -1424,38 +1423,21 @@
*totalrate_nocoef = 0;
#endif // CONFIG_SUPERTX
} else {
- if (bsize >= BLOCK_8X8 || unify_bsize) {
- if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
- av1_rd_pick_inter_mode_sb_seg_skip(cpi, tile_data, x, mi_row, mi_col,
- rd_cost, bsize, ctx, best_rd);
+ if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+ av1_rd_pick_inter_mode_sb_seg_skip(cpi, tile_data, x, mi_row, mi_col,
+ rd_cost, bsize, ctx, best_rd);
#if CONFIG_SUPERTX
- *totalrate_nocoef = rd_cost->rate;
+ *totalrate_nocoef = rd_cost->rate;
#endif // CONFIG_SUPERTX
- } else {
- av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost,
-#if CONFIG_SUPERTX
- totalrate_nocoef,
-#endif // CONFIG_SUPERTX
- bsize, ctx, best_rd);
-#if CONFIG_SUPERTX
- assert(*totalrate_nocoef >= 0);
-#endif // CONFIG_SUPERTX
- }
} else {
- if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
- // The decoder rejects sub8x8 partitions when SEG_LVL_SKIP is set.
- rd_cost->rate = INT_MAX;
- } else {
- av1_rd_pick_inter_mode_sub8x8(cpi, tile_data, x, mi_row, mi_col,
- rd_cost,
+ av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost,
#if CONFIG_SUPERTX
- totalrate_nocoef,
+ totalrate_nocoef,
#endif // CONFIG_SUPERTX
- bsize, ctx, best_rd);
+ bsize, ctx, best_rd);
#if CONFIG_SUPERTX
- assert(*totalrate_nocoef >= 0);
+ assert(*totalrate_nocoef >= 0);
#endif // CONFIG_SUPERTX
- }
}
}
@@ -1525,7 +1507,6 @@
const MB_MODE_INFO *const mbmi = &mi->mbmi;
const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
const BLOCK_SIZE bsize = mbmi->sb_type;
- const int unify_bsize = CONFIG_CB4X4;
#if CONFIG_DELTA_Q
// delta quant applies to both intra and inter
@@ -1697,79 +1678,54 @@
if (inter_block &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
int16_t mode_ctx;
- if (bsize >= BLOCK_8X8 || unify_bsize) {
- const PREDICTION_MODE mode = mbmi->mode;
+ const PREDICTION_MODE mode = mbmi->mode;
#if CONFIG_EXT_INTER
- if (has_second_ref(mbmi)) {
- mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]];
- ++counts->inter_compound_mode[mode_ctx][INTER_COMPOUND_OFFSET(mode)];
- } else {
-#endif // CONFIG_EXT_INTER
- mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context,
- mbmi->ref_frame, bsize, -1);
- update_inter_mode_stats(counts, mode, mode_ctx);
-#if CONFIG_EXT_INTER
- }
-#endif // CONFIG_EXT_INTER
-
-#if CONFIG_EXT_INTER
- if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
-#else
- if (mbmi->mode == NEWMV) {
-#endif
- uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
- int idx;
-
- for (idx = 0; idx < 2; ++idx) {
- if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
- uint8_t drl_ctx =
- av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
- ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx];
-
- if (mbmi->ref_mv_idx == idx) break;
- }
- }
- }
-
-#if CONFIG_EXT_INTER
- if (have_nearmv_in_inter_mode(mbmi->mode)) {
-#else
- if (mbmi->mode == NEARMV) {
-#endif
- uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
- int idx;
-
- for (idx = 1; idx < 3; ++idx) {
- if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
- uint8_t drl_ctx =
- av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
- ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx - 1];
-
- if (mbmi->ref_mv_idx == idx - 1) break;
- }
- }
- }
+ if (has_second_ref(mbmi)) {
+ mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]];
+ ++counts->inter_compound_mode[mode_ctx][INTER_COMPOUND_OFFSET(mode)];
} else {
- const int num_4x4_w = num_4x4_blocks_wide_lookup[bsize];
- const int num_4x4_h = num_4x4_blocks_high_lookup[bsize];
- int idx, idy;
- for (idy = 0; idy < 2; idy += num_4x4_h) {
- for (idx = 0; idx < 2; idx += num_4x4_w) {
- const int j = idy * 2 + idx;
- const PREDICTION_MODE b_mode = mi->bmi[j].as_mode;
-#if CONFIG_EXT_INTER
- if (has_second_ref(mbmi)) {
- mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]];
- ++counts->inter_compound_mode[mode_ctx]
- [INTER_COMPOUND_OFFSET(b_mode)];
- } else {
#endif // CONFIG_EXT_INTER
- mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context,
- mbmi->ref_frame, bsize, j);
- update_inter_mode_stats(counts, b_mode, mode_ctx);
+ mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context,
+ mbmi->ref_frame, bsize, -1);
+ update_inter_mode_stats(counts, mode, mode_ctx);
#if CONFIG_EXT_INTER
- }
+ }
#endif // CONFIG_EXT_INTER
+
+#if CONFIG_EXT_INTER
+ if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
+#else
+ if (mbmi->mode == NEWMV) {
+#endif
+ uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
+ int idx;
+
+ for (idx = 0; idx < 2; ++idx) {
+ if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
+ uint8_t drl_ctx =
+ av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
+ ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx];
+
+ if (mbmi->ref_mv_idx == idx) break;
+ }
+ }
+ }
+
+#if CONFIG_EXT_INTER
+ if (have_nearmv_in_inter_mode(mbmi->mode)) {
+#else
+ if (mbmi->mode == NEARMV) {
+#endif
+ uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
+ int idx;
+
+ for (idx = 1; idx < 3; ++idx) {
+ if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
+ uint8_t drl_ctx =
+ av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
+ ++counts->drl_mode[drl_ctx][mbmi->ref_mv_idx != idx - 1];
+
+ if (mbmi->ref_mv_idx == idx - 1) break;
}
}
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 57c8bc8..caf17b5 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -378,28 +378,6 @@
#endif // CONFIG_EXT_INTER
};
-static const REF_DEFINITION av1_ref_order[MAX_REFS] = {
- { { LAST_FRAME, NONE_FRAME } },
-#if CONFIG_EXT_REFS
- { { LAST2_FRAME, NONE_FRAME } }, { { LAST3_FRAME, NONE_FRAME } },
- { { BWDREF_FRAME, NONE_FRAME } },
-#endif // CONFIG_EXT_REFS
- { { GOLDEN_FRAME, NONE_FRAME } }, { { ALTREF_FRAME, NONE_FRAME } },
-
- { { LAST_FRAME, ALTREF_FRAME } },
-#if CONFIG_EXT_REFS
- { { LAST2_FRAME, ALTREF_FRAME } }, { { LAST3_FRAME, ALTREF_FRAME } },
-#endif // CONFIG_EXT_REFS
- { { GOLDEN_FRAME, ALTREF_FRAME } },
-
-#if CONFIG_EXT_REFS
- { { LAST_FRAME, BWDREF_FRAME } }, { { LAST2_FRAME, BWDREF_FRAME } },
- { { LAST3_FRAME, BWDREF_FRAME } }, { { GOLDEN_FRAME, BWDREF_FRAME } },
-#endif // CONFIG_EXT_REFS
-
- { { INTRA_FRAME, NONE_FRAME } },
-};
-
#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE
static INLINE int write_uniform_cost(int n, int v) {
const int l = get_unsigned_bits(n);
@@ -5076,274 +5054,6 @@
}
#endif // CONFIG_EXT_INTER && (CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT)
-static int set_and_cost_bmi_mvs(
- const AV1_COMP *const cpi, MACROBLOCK *x, MACROBLOCKD *xd, int i,
- PREDICTION_MODE mode, int_mv this_mv[2],
- int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME],
- int_mv seg_mvs[TOTAL_REFS_PER_FRAME],
-#if CONFIG_EXT_INTER
- int_mv compound_seg_newmvs[2],
-#endif // CONFIG_EXT_INTER
- int_mv *best_ref_mv[2], const int *mvjcost, int *mvcost[2], int mi_row,
- int mi_col) {
- MODE_INFO *const mic = xd->mi[0];
- const MB_MODE_INFO *const mbmi = &mic->mbmi;
- const MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
- int thismvcost = 0;
- int idx, idy;
- const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type];
- const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
- const int is_compound = has_second_ref(mbmi);
- int mode_ctx;
- (void)mi_row;
- (void)mi_col;
-
- switch (mode) {
- case NEWMV: this_mv[0].as_int = seg_mvs[mbmi->ref_frame[0]].as_int;
-#if CONFIG_EXT_INTER
- if (!cpi->common.allow_high_precision_mv)
- lower_mv_precision(&this_mv[0].as_mv, 0);
-#endif // CONFIG_EXT_INTER
-
- for (idx = 0; idx < 1 + is_compound; ++idx) {
- this_mv[idx] = seg_mvs[mbmi->ref_frame[idx]];
- av1_set_mvcost(x, mbmi->ref_frame[idx], idx, mbmi->ref_mv_idx);
- thismvcost +=
- av1_mv_bit_cost(&this_mv[idx].as_mv, &best_ref_mv[idx]->as_mv,
- x->nmvjointcost, x->mvcost, MV_COST_WEIGHT_SUB);
- }
- (void)mvjcost;
- (void)mvcost;
- break;
- case NEARMV:
- case NEARESTMV:
- this_mv[0].as_int = frame_mv[mode][mbmi->ref_frame[0]].as_int;
- if (is_compound)
- this_mv[1].as_int = frame_mv[mode][mbmi->ref_frame[1]].as_int;
- break;
- case ZEROMV: {
- int ref;
- for (ref = 0; ref < 1 + is_compound; ++ref) {
-#if CONFIG_GLOBAL_MOTION
- this_mv[ref].as_int =
- gm_get_motion_vector(
- &cpi->common.global_motion[mbmi->ref_frame[ref]],
- cpi->common.allow_high_precision_mv, mbmi->sb_type, mi_col,
- mi_row, i)
- .as_int;
-#else
- this_mv[ref].as_int = 0;
-#endif // CONFIG_GLOBAL_MOTION
- }
- break;
- }
-#if CONFIG_EXT_INTER
- case NEW_NEWMV:
- if (compound_seg_newmvs[0].as_int == INVALID_MV ||
- compound_seg_newmvs[1].as_int == INVALID_MV) {
- this_mv[0].as_int = seg_mvs[mbmi->ref_frame[0]].as_int;
- this_mv[1].as_int = seg_mvs[mbmi->ref_frame[1]].as_int;
- } else {
- this_mv[0].as_int = compound_seg_newmvs[0].as_int;
- this_mv[1].as_int = compound_seg_newmvs[1].as_int;
- }
- if (!cpi->common.allow_high_precision_mv)
- lower_mv_precision(&this_mv[0].as_mv, 0);
- if (!cpi->common.allow_high_precision_mv)
- lower_mv_precision(&this_mv[1].as_mv, 0);
- av1_set_mvcost(x, mbmi->ref_frame[0], 0, mbmi->ref_mv_idx);
- thismvcost += av1_mv_bit_cost(&this_mv[0].as_mv, &best_ref_mv[0]->as_mv,
- mvjcost, mvcost, MV_COST_WEIGHT_SUB);
- av1_set_mvcost(x, mbmi->ref_frame[1], 1, mbmi->ref_mv_idx);
- thismvcost += av1_mv_bit_cost(&this_mv[1].as_mv, &best_ref_mv[1]->as_mv,
- mvjcost, mvcost, MV_COST_WEIGHT_SUB);
- break;
- case NEW_NEARMV:
- case NEW_NEARESTMV:
- this_mv[0].as_int = seg_mvs[mbmi->ref_frame[0]].as_int;
- if (!cpi->common.allow_high_precision_mv)
- lower_mv_precision(&this_mv[0].as_mv, 0);
- av1_set_mvcost(x, mbmi->ref_frame[0], 0, mbmi->ref_mv_idx);
- thismvcost += av1_mv_bit_cost(&this_mv[0].as_mv, &best_ref_mv[0]->as_mv,
- mvjcost, mvcost, MV_COST_WEIGHT_SUB);
- this_mv[1].as_int = frame_mv[mode][mbmi->ref_frame[1]].as_int;
- break;
- case NEAR_NEWMV:
- case NEAREST_NEWMV:
- this_mv[0].as_int = frame_mv[mode][mbmi->ref_frame[0]].as_int;
- this_mv[1].as_int = seg_mvs[mbmi->ref_frame[1]].as_int;
- if (!cpi->common.allow_high_precision_mv)
- lower_mv_precision(&this_mv[1].as_mv, 0);
- av1_set_mvcost(x, mbmi->ref_frame[1], 1, mbmi->ref_mv_idx);
- thismvcost += av1_mv_bit_cost(&this_mv[1].as_mv, &best_ref_mv[1]->as_mv,
- mvjcost, mvcost, MV_COST_WEIGHT_SUB);
- break;
- case NEAREST_NEARESTMV:
- case NEAR_NEARMV:
- this_mv[0].as_int = frame_mv[mode][mbmi->ref_frame[0]].as_int;
- this_mv[1].as_int = frame_mv[mode][mbmi->ref_frame[1]].as_int;
- break;
- case ZERO_ZEROMV:
-#if CONFIG_GLOBAL_MOTION
- this_mv[0].as_int =
- gm_get_motion_vector(&cpi->common.global_motion[mbmi->ref_frame[0]],
- cpi->common.allow_high_precision_mv,
- mbmi->sb_type, mi_col, mi_row, i)
- .as_int;
- this_mv[1].as_int =
- gm_get_motion_vector(&cpi->common.global_motion[mbmi->ref_frame[1]],
- cpi->common.allow_high_precision_mv,
- mbmi->sb_type, mi_col, mi_row, i)
- .as_int;
-#else
- this_mv[0].as_int = 0;
- this_mv[1].as_int = 0;
-#endif // CONFIG_GLOBAL_MOTION
- break;
-#endif // CONFIG_EXT_INTER
- default: break;
- }
-
- mic->bmi[i].as_mv[0].as_int = this_mv[0].as_int;
- if (is_compound) mic->bmi[i].as_mv[1].as_int = this_mv[1].as_int;
-
- mic->bmi[i].as_mode = mode;
-
- if (mode == NEWMV) {
- mic->bmi[i].pred_mv[0].as_int =
- mbmi_ext->ref_mvs[mbmi->ref_frame[0]][0].as_int;
- if (is_compound)
- mic->bmi[i].pred_mv[1].as_int =
- mbmi_ext->ref_mvs[mbmi->ref_frame[1]][0].as_int;
- } else {
- mic->bmi[i].pred_mv[0].as_int = this_mv[0].as_int;
- if (is_compound) mic->bmi[i].pred_mv[1].as_int = this_mv[1].as_int;
- }
-
- for (idy = 0; idy < num_4x4_blocks_high; ++idy)
- for (idx = 0; idx < num_4x4_blocks_wide; ++idx)
- memmove(&mic->bmi[i + idy * 2 + idx], &mic->bmi[i], sizeof(mic->bmi[i]));
-
-#if CONFIG_EXT_INTER
- if (is_compound)
- mode_ctx = mbmi_ext->compound_mode_context[mbmi->ref_frame[0]];
- else
-#endif // CONFIG_EXT_INTER
- mode_ctx = av1_mode_context_analyzer(mbmi_ext->mode_context,
- mbmi->ref_frame, mbmi->sb_type, i);
- return cost_mv_ref(cpi, mode, mode_ctx) + thismvcost;
-}
-
-static int64_t encode_inter_mb_segment_sub8x8(
- const AV1_COMP *const cpi, MACROBLOCK *x, int64_t best_yrd, int i,
- int *labelyrate, int64_t *distortion, int64_t *sse, ENTROPY_CONTEXT *ta,
- ENTROPY_CONTEXT *tl, int ir, int ic, int mi_row, int mi_col) {
- const AV1_COMMON *const cm = &cpi->common;
- MACROBLOCKD *xd = &x->e_mbd;
- struct macroblockd_plane *const pd = &xd->plane[0];
- struct macroblock_plane *const p = &x->plane[0];
- MODE_INFO *const mi = xd->mi[0];
- const BLOCK_SIZE plane_bsize = get_plane_block_size(mi->mbmi.sb_type, pd);
- const int txb_width = max_block_wide(xd, plane_bsize, 0);
- const int txb_height = max_block_high(xd, plane_bsize, 0);
- const int width = block_size_wide[plane_bsize];
- const int height = block_size_high[plane_bsize];
- int idx, idy;
- const uint8_t *const src =
- &p->src.buf[av1_raster_block_offset(BLOCK_8X8, i, p->src.stride)];
- uint8_t *const dst =
- &pd->dst.buf[av1_raster_block_offset(BLOCK_8X8, i, pd->dst.stride)];
- int64_t thisdistortion = 0, thissse = 0;
- int thisrate = 0;
- TX_SIZE tx_size = mi->mbmi.tx_size;
- TX_TYPE tx_type = get_tx_type(PLANE_TYPE_Y, xd, i, tx_size);
- const int num_4x4_w = tx_size_wide_unit[tx_size];
- const int num_4x4_h = tx_size_high_unit[tx_size];
-#if !CONFIG_PVQ
- const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
-#else
- (void)cpi;
- (void)ta;
- (void)tl;
- (void)tx_type;
-#endif // !CONFIG_PVQ
-
-#if CONFIG_EXT_TX && CONFIG_RECT_TX
- assert(IMPLIES(xd->lossless[mi->mbmi.segment_id], tx_size == TX_4X4));
- assert(IMPLIES(!xd->lossless[mi->mbmi.segment_id],
- tx_size == max_txsize_rect_lookup[mi->mbmi.sb_type]));
-#else
- assert(tx_size == TX_4X4);
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
-
- assert(tx_type == DCT_DCT);
-
- av1_build_inter_predictor_sub8x8(cm, xd, 0, i, ir, ic, mi_row, mi_col);
-
-#if CONFIG_HIGHBITDEPTH
- if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
- aom_highbd_subtract_block(
- height, width, av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
- 8, src, p->src.stride, dst, pd->dst.stride, xd->bd);
- } else {
- aom_subtract_block(height, width,
- av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
- 8, src, p->src.stride, dst, pd->dst.stride);
- }
-#else
- aom_subtract_block(height, width,
- av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
- 8, src, p->src.stride, dst, pd->dst.stride);
-#endif // CONFIG_HIGHBITDEPTH
-
- for (idy = 0; idy < txb_height; idy += num_4x4_h) {
- for (idx = 0; idx < txb_width; idx += num_4x4_w) {
- int64_t dist, ssz, rd, rd1, rd2;
- int coeff_ctx;
- const int k = i + (idy * 2 + idx);
- const int block = av1_raster_order_to_block_index(tx_size, k);
- assert(IMPLIES(tx_size == TX_4X8 || tx_size == TX_8X4,
- idx == 0 && idy == 0));
- coeff_ctx = combine_entropy_contexts(*(ta + (k & 1)), *(tl + (k >> 1)));
- av1_xform_quant(cm, x, 0, block, idy + (i >> 1), idx + (i & 0x01),
- BLOCK_8X8, tx_size, coeff_ctx, AV1_XFORM_QUANT_FP);
- av1_optimize_b(cm, x, 0, block, tx_size, coeff_ctx);
- av1_dist_block(cpi, x, 0, BLOCK_8X8, block, idy + (i >> 1),
- idx + (i & 0x1), tx_size, &dist, &ssz,
- OUTPUT_HAS_PREDICTED_PIXELS);
- thisdistortion += dist;
- thissse += ssz;
-#if !CONFIG_PVQ
- thisrate +=
- av1_cost_coeffs(cpi, x, 0, block, tx_size, scan_order, (ta + (k & 1)),
- (tl + (k >> 1)), cpi->sf.use_fast_coef_costing);
- *(ta + (k & 1)) = !(p->eobs[block] == 0);
- *(tl + (k >> 1)) = !(p->eobs[block] == 0);
-#else
- thisrate += x->rate;
-#endif // !CONFIG_PVQ
-#if CONFIG_EXT_TX
- if (tx_size == TX_8X4) {
- *(ta + (k & 1) + 1) = *(ta + (k & 1));
- }
- if (tx_size == TX_4X8) {
- *(tl + (k >> 1) + 1) = *(tl + (k >> 1));
- }
-#endif // CONFIG_EXT_TX
- rd1 = RDCOST(x->rdmult, x->rddiv, thisrate, thisdistortion);
- rd2 = RDCOST(x->rdmult, x->rddiv, 0, thissse);
- rd = AOMMIN(rd1, rd2);
- if (rd >= best_yrd) return INT64_MAX;
- }
- }
-
- *distortion = thisdistortion;
- *labelyrate = thisrate;
- *sse = thissse;
-
- return RDCOST(x->rdmult, x->rddiv, *labelyrate, *distortion);
-}
-
typedef struct {
int eobs;
int brate;
@@ -5391,30 +5101,6 @@
(mv->col >> 3) > mv_limits->col_max;
}
-static INLINE void mi_buf_shift(MACROBLOCK *x, int i) {
- MB_MODE_INFO *const mbmi = &x->e_mbd.mi[0]->mbmi;
- struct macroblock_plane *const p = &x->plane[0];
- struct macroblockd_plane *const pd = &x->e_mbd.plane[0];
-
- p->src.buf =
- &p->src.buf[av1_raster_block_offset(BLOCK_8X8, i, p->src.stride)];
- assert(((intptr_t)pd->pre[0].buf & 0x7) == 0);
- pd->pre[0].buf =
- &pd->pre[0].buf[av1_raster_block_offset(BLOCK_8X8, i, pd->pre[0].stride)];
- if (has_second_ref(mbmi))
- pd->pre[1].buf =
- &pd->pre[1]
- .buf[av1_raster_block_offset(BLOCK_8X8, i, pd->pre[1].stride)];
-}
-
-static INLINE void mi_buf_restore(MACROBLOCK *x, struct buf_2d orig_src,
- struct buf_2d orig_pre[2]) {
- MB_MODE_INFO *mbmi = &x->e_mbd.mi[0]->mbmi;
- x->plane[0].src = orig_src;
- x->e_mbd.plane[0].pre[0] = orig_pre[0];
- if (has_second_ref(mbmi)) x->e_mbd.plane[0].pre[1] = orig_pre[1];
-}
-
// Check if NEARESTMV/NEARMV/ZEROMV is the cheapest way encode zero motion.
// TODO(aconverse): Find out if this is still productive then clean up or remove
static int check_best_zero_mv(
@@ -5781,911 +5467,6 @@
}
}
-#if !CONFIG_EXT_INTER
-static void update_mv_search_and_seg_mvs(
- int *const run_mv_search, int_mv *const seg_mvs, int has_second_rf,
- const MV_REFERENCE_FRAME *const ref_frame,
- const SEG_RDSTAT *const ref_rdstat, int_mv *const bsi_ref_mv[2]) {
- if (has_second_rf) {
- if (seg_mvs[ref_frame[0]].as_int == ref_rdstat->mvs[0].as_int &&
- ref_rdstat->mvs[0].as_int != INVALID_MV)
- if (bsi_ref_mv[0]->as_int == ref_rdstat->pred_mv[0].as_int)
- --*run_mv_search;
-
- if (seg_mvs[ref_frame[1]].as_int == ref_rdstat->mvs[1].as_int &&
- ref_rdstat->mvs[1].as_int != INVALID_MV)
- if (bsi_ref_mv[1]->as_int == ref_rdstat->pred_mv[1].as_int)
- --*run_mv_search;
- } else {
- if (bsi_ref_mv[0]->as_int == ref_rdstat->pred_mv[0].as_int &&
- ref_rdstat->mvs[0].as_int != INVALID_MV) {
- *run_mv_search = 0;
- seg_mvs[ref_frame[0]].as_int = ref_rdstat->mvs[0].as_int;
- }
- }
-}
-#endif // !CONFIG_EXT_INTER
-
-static int64_t rd_pick_inter_best_sub8x8_mode(
- const AV1_COMP *const cpi, MACROBLOCK *x, int_mv *best_ref_mv,
- int_mv *second_best_ref_mv, int64_t best_rd, int *returntotrate,
- int *returnyrate, int64_t *returndistortion, int *skippable, int64_t *psse,
- int mvthresh, int_mv seg_mvs[4][TOTAL_REFS_PER_FRAME],
-#if CONFIG_EXT_INTER
- int_mv compound_seg_newmvs[4][2],
-#endif // CONFIG_EXT_INTER
- BEST_SEG_INFO *bsi_buf, int filter_idx, int mi_row, int mi_col) {
- BEST_SEG_INFO *bsi = bsi_buf + filter_idx;
- int_mv tmp_ref_mv[2];
- MACROBLOCKD *xd = &x->e_mbd;
- MODE_INFO *mi = xd->mi[0];
- MB_MODE_INFO *mbmi = &mi->mbmi;
- int mode_idx;
- int k, br = 0, idx, idy;
- int64_t bd = 0, block_sse = 0;
- PREDICTION_MODE this_mode;
- const AV1_COMMON *cm = &cpi->common;
- struct macroblock_plane *const p = &x->plane[0];
- struct macroblockd_plane *const pd = &xd->plane[0];
- const int label_count = 4;
- int64_t this_segment_rd = 0;
- int label_mv_thresh;
- int segmentyrate = 0;
- const BLOCK_SIZE bsize = mbmi->sb_type;
- const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize];
- const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize];
-#if CONFIG_CHROMA_2X2
- ENTROPY_CONTEXT t_above[4], t_left[4];
-#else
- ENTROPY_CONTEXT t_above[2], t_left[2];
-#endif // CONFIG_CHROMA_2X2
- int subpelmv = 1, have_ref = 0;
- const int has_second_rf = has_second_ref(mbmi);
- const int inter_mode_mask = cpi->sf.inter_mode_mask[bsize];
- MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
-#if CONFIG_PVQ
- od_rollback_buffer pre_buf;
-
- od_encode_checkpoint(&x->daala_enc, &pre_buf);
-#endif // CONFIG_PVQ
-#if CONFIG_EXT_TX && CONFIG_RECT_TX
- mbmi->tx_size =
- xd->lossless[mbmi->segment_id] ? TX_4X4 : max_txsize_rect_lookup[bsize];
-#else
- mbmi->tx_size = TX_4X4;
-#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
-
- av1_zero(*bsi);
-
- bsi->segment_rd = best_rd;
- bsi->ref_mv[0] = best_ref_mv;
- bsi->ref_mv[1] = second_best_ref_mv;
- bsi->mvp.as_int = best_ref_mv->as_int;
- bsi->mvthresh = mvthresh;
-
- for (idx = 0; idx < 4; ++idx) bsi->modes[idx] = ZEROMV;
-
- for (idx = 0; idx < 4; ++idx) {
- for (k = NEARESTMV; k <= NEWMV; ++k) {
- bsi->rdstat[idx][INTER_OFFSET(k)].pred_mv[0].as_int = INVALID_MV;
- bsi->rdstat[idx][INTER_OFFSET(k)].pred_mv[1].as_int = INVALID_MV;
-
- bsi->rdstat[idx][INTER_OFFSET(k)].mvs[0].as_int = INVALID_MV;
- bsi->rdstat[idx][INTER_OFFSET(k)].mvs[1].as_int = INVALID_MV;
- }
- }
-
- memcpy(t_above, pd->above_context, sizeof(t_above));
- memcpy(t_left, pd->left_context, sizeof(t_left));
-
- // 64 makes this threshold really big effectively
- // making it so that we very rarely check mvs on
- // segments. setting this to 1 would make mv thresh
- // roughly equal to what it is for macroblocks
- label_mv_thresh = 1 * bsi->mvthresh / label_count;
-
- // Segmentation method overheads
- for (idy = 0; idy < 2; idy += num_4x4_blocks_high) {
- for (idx = 0; idx < 2; idx += num_4x4_blocks_wide) {
- // TODO(jingning,rbultje): rewrite the rate-distortion optimization
- // loop for 4x4/4x8/8x4 block coding. to be replaced with new rd loop
- int_mv mode_mv[MB_MODE_COUNT][2];
- int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME];
- PREDICTION_MODE mode_selected = ZEROMV;
- int64_t new_best_rd = INT64_MAX;
- const int index = idy * 2 + idx;
- int ref;
- CANDIDATE_MV ref_mv_stack[2][MAX_REF_MV_STACK_SIZE];
- uint8_t ref_mv_count[2];
-#if CONFIG_EXT_INTER
- int_mv ref_mvs_sub8x8[2][2];
-#endif // CONFIG_EXT_INTER
-#if CONFIG_PVQ
- od_rollback_buffer idx_buf, post_buf;
- od_encode_checkpoint(&x->daala_enc, &idx_buf);
- od_encode_checkpoint(&x->daala_enc, &post_buf);
-#endif // CONFIG_PVQ
-
- for (ref = 0; ref < 1 + has_second_rf; ++ref) {
- const MV_REFERENCE_FRAME frame = mbmi->ref_frame[ref];
-#if CONFIG_EXT_INTER
- int_mv mv_ref_list[MAX_MV_REF_CANDIDATES];
- av1_update_mv_context(cm, xd, mi, frame, mv_ref_list, index, mi_row,
- mi_col, NULL);
-#endif // CONFIG_EXT_INTER
-#if CONFIG_GLOBAL_MOTION
- frame_mv[ZEROMV][frame].as_int =
- gm_get_motion_vector(&cm->global_motion[frame],
- cm->allow_high_precision_mv, mbmi->sb_type,
- mi_col, mi_row, index)
- .as_int;
-#else // CONFIG_GLOBAL_MOTION
- frame_mv[ZEROMV][frame].as_int = 0;
-#endif // CONFIG_GLOBAL_MOTION
- av1_append_sub8x8_mvs_for_idx(cm, xd, index, ref, mi_row, mi_col,
- ref_mv_stack[ref], &ref_mv_count[ref],
-#if CONFIG_EXT_INTER
- mv_ref_list,
-#endif // CONFIG_EXT_INTER
- &frame_mv[NEARESTMV][frame],
- &frame_mv[NEARMV][frame]);
-
- tmp_ref_mv[ref] = frame_mv[NEARESTMV][mbmi->ref_frame[ref]];
- lower_mv_precision(&tmp_ref_mv[ref].as_mv, cm->allow_high_precision_mv);
- bsi->ref_mv[ref] = &tmp_ref_mv[ref];
- mbmi_ext->ref_mvs[frame][0] = tmp_ref_mv[ref];
-
-#if CONFIG_EXT_INTER
- mv_ref_list[0].as_int = frame_mv[NEARESTMV][frame].as_int;
- mv_ref_list[1].as_int = frame_mv[NEARMV][frame].as_int;
- av1_find_best_ref_mvs(cm->allow_high_precision_mv, mv_ref_list,
- &ref_mvs_sub8x8[0][ref], &ref_mvs_sub8x8[1][ref]);
-
- if (has_second_rf) {
-#if CONFIG_GLOBAL_MOTION
- frame_mv[ZERO_ZEROMV][frame].as_int =
- gm_get_motion_vector(&cm->global_motion[frame],
- cm->allow_high_precision_mv, mbmi->sb_type,
- mi_col, mi_row, index)
- .as_int;
-#else
- frame_mv[ZERO_ZEROMV][frame].as_int = 0;
-#endif // CONFIG_GLOBAL_MOTION
- frame_mv[NEAREST_NEARESTMV][frame].as_int =
- frame_mv[NEARESTMV][frame].as_int;
-
- if (ref == 0) {
- frame_mv[NEAREST_NEWMV][frame].as_int =
- frame_mv[NEARESTMV][frame].as_int;
- frame_mv[NEAR_NEWMV][frame].as_int = frame_mv[NEARMV][frame].as_int;
- frame_mv[NEAR_NEARMV][frame].as_int =
- frame_mv[NEARMV][frame].as_int;
- } else if (ref == 1) {
- frame_mv[NEW_NEARESTMV][frame].as_int =
- frame_mv[NEARESTMV][frame].as_int;
- frame_mv[NEW_NEARMV][frame].as_int = frame_mv[NEARMV][frame].as_int;
- frame_mv[NEAR_NEARMV][frame].as_int =
- frame_mv[NEARMV][frame].as_int;
- }
- }
-#endif // CONFIG_EXT_INTER
- }
-
-// search for the best motion vector on this segment
-#if CONFIG_EXT_INTER
- for (this_mode = (has_second_rf ? NEAREST_NEARESTMV : NEARESTMV);
- this_mode <= (has_second_rf ? NEW_NEWMV : NEWMV); ++this_mode)
-#else
- for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode)
-#endif // CONFIG_EXT_INTER
- {
- const struct buf_2d orig_src = x->plane[0].src;
- struct buf_2d orig_pre[2];
- // This flag controls if the motion estimation will kick off. When it
- // is set to a non-zero value, the encoder will force motion estimation.
- int run_mv_search = 0;
-
- mode_idx = INTER_OFFSET(this_mode);
-#if CONFIG_EXT_INTER
- for (ref = 0; ref < 1 + has_second_rf; ++ref)
- bsi->ref_mv[ref]->as_int = ref_mvs_sub8x8[0][ref].as_int;
-#endif // CONFIG_EXT_INTER
- bsi->rdstat[index][mode_idx].brdcost = INT64_MAX;
- if (!(inter_mode_mask & (1 << this_mode))) continue;
-
- run_mv_search = 2;
-#if !CONFIG_EXT_INTER
- if (filter_idx > 0 && this_mode == NEWMV) {
- const BEST_SEG_INFO *ref_bsi = bsi_buf;
- const SEG_RDSTAT *ref_rdstat = &ref_bsi->rdstat[index][mode_idx];
-
- update_mv_search_and_seg_mvs(&run_mv_search, seg_mvs[index],
- has_second_rf, mbmi->ref_frame,
- ref_rdstat, bsi->ref_mv);
-
- if (run_mv_search != 0 && filter_idx > 1) {
- ref_bsi = bsi_buf + 1;
- ref_rdstat = &ref_bsi->rdstat[index][mode_idx];
- run_mv_search = 2;
- update_mv_search_and_seg_mvs(&run_mv_search, seg_mvs[index],
- has_second_rf, mbmi->ref_frame,
- ref_rdstat, bsi->ref_mv);
- }
- }
-#endif // !CONFIG_EXT_INTER
-
-#if CONFIG_GLOBAL_MOTION
- if (cm->global_motion[mbmi->ref_frame[0]].wmtype == IDENTITY &&
- (!has_second_rf ||
- cm->global_motion[mbmi->ref_frame[1]].wmtype == IDENTITY))
-#endif // CONFIG_GLOBAL_MOTION
-
- if (!check_best_zero_mv(cpi, mbmi_ext->mode_context,
-#if CONFIG_EXT_INTER
- mbmi_ext->compound_mode_context,
-#endif // CONFIG_EXT_INTER
- frame_mv, this_mode, mbmi->ref_frame, bsize,
- index, mi_row, mi_col))
- continue;
-
- memcpy(orig_pre, pd->pre, sizeof(orig_pre));
- memcpy(bsi->rdstat[index][mode_idx].ta, t_above,
- sizeof(bsi->rdstat[index][mode_idx].ta));
- memcpy(bsi->rdstat[index][mode_idx].tl, t_left,
- sizeof(bsi->rdstat[index][mode_idx].tl));
-#if CONFIG_PVQ
- od_encode_rollback(&x->daala_enc, &idx_buf);
-#endif // CONFIG_PVQ
-
- // motion search for newmv (single predictor case only)
- if (!has_second_rf &&
-#if CONFIG_EXT_INTER
- have_newmv_in_inter_mode(this_mode) &&
- (seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV)
-#else
- this_mode == NEWMV &&
- (seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV ||
- run_mv_search)
-#endif // CONFIG_EXT_INTER
- ) {
- int step_param = 0;
- int bestsme = INT_MAX;
- int sadpb = x->sadperbit4;
- MV mvp_full;
- int max_mv;
- int cost_list[5];
- MvLimits tmp_mv_limits = x->mv_limits;
-
- /* Is the best so far sufficiently good that we cant justify doing
- * and new motion search. */
- if (new_best_rd < label_mv_thresh) break;
-
-#if CONFIG_EXT_INTER
- bsi->mvp.as_int = bsi->ref_mv[0]->as_int;
-#else
-// use previous block's result as next block's MV predictor.
-#endif // CONFIG_EXT_INTER
- max_mv = (index == 0) ? (int)x->max_mv_context[mbmi->ref_frame[0]]
- : AOMMAX(abs(bsi->mvp.as_mv.row),
- abs(bsi->mvp.as_mv.col)) >>
- 3;
-
- if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) {
- // Take wtd average of the step_params based on the last frame's
- // max mv magnitude and the best ref mvs of the current block for
- // the given reference.
- step_param =
- (av1_init_search_range(max_mv) + cpi->mv_step_param) / 2;
- } else {
- step_param = cpi->mv_step_param;
- }
-
- mvp_full.row = bsi->ref_mv[0]->as_mv.row >> 3;
- mvp_full.col = bsi->ref_mv[0]->as_mv.col >> 3;
-
- if (cpi->sf.adaptive_motion_search) {
- mvp_full.row = x->pred_mv[mbmi->ref_frame[0]].row >> 3;
- mvp_full.col = x->pred_mv[mbmi->ref_frame[0]].col >> 3;
- step_param = AOMMAX(step_param, 8);
- }
-
- // adjust src pointer for this block
- mi_buf_shift(x, index);
-
- av1_set_mv_search_range(&x->mv_limits, &bsi->ref_mv[0]->as_mv);
-
- x->best_mv.as_int = x->second_best_mv.as_int = INVALID_MV;
-
- av1_set_mvcost(x, mbmi->ref_frame[0], 0, mbmi->ref_mv_idx);
- bestsme = av1_full_pixel_search(
- cpi, x, bsize, &mvp_full, step_param, sadpb,
- cpi->sf.mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL,
- &bsi->ref_mv[0]->as_mv, INT_MAX, 1);
-
- x->mv_limits = tmp_mv_limits;
-
- if (bestsme < INT_MAX) {
- int distortion;
- if (cpi->sf.use_upsampled_references) {
- int best_mv_var;
- const int try_second =
- x->second_best_mv.as_int != INVALID_MV &&
- x->second_best_mv.as_int != x->best_mv.as_int;
- const int pw = block_size_wide[bsize];
- const int ph = block_size_high[bsize];
- // Use up-sampled reference frames.
- struct buf_2d backup_pred = pd->pre[0];
- const YV12_BUFFER_CONFIG *upsampled_ref =
- get_upsampled_ref(cpi, mbmi->ref_frame[0]);
-
- // Set pred for Y plane
- setup_pred_plane(
- &pd->pre[0], bsize, upsampled_ref->y_buffer,
- upsampled_ref->y_crop_width, upsampled_ref->y_crop_height,
- upsampled_ref->y_stride, (mi_row << 3), (mi_col << 3), NULL,
- pd->subsampling_x, pd->subsampling_y);
-
- // adjust pred pointer for this block
- pd->pre[0].buf =
- &pd->pre[0].buf[(av1_raster_block_offset(BLOCK_8X8, index,
- pd->pre[0].stride))
- << 3];
-
- best_mv_var = cpi->find_fractional_mv_step(
- x, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv,
- x->errorperbit, &cpi->fn_ptr[bsize],
- cpi->sf.mv.subpel_force_stop,
- cpi->sf.mv.subpel_iters_per_step,
- cond_cost_list(cpi, cost_list), x->nmvjointcost, x->mvcost,
- &distortion, &x->pred_sse[mbmi->ref_frame[0]], NULL,
-#if CONFIG_EXT_INTER
- NULL, 0, 0,
-#endif
- pw, ph, 1);
-
- if (try_second) {
- int this_var;
- MV best_mv = x->best_mv.as_mv;
- const MV ref_mv = bsi->ref_mv[0]->as_mv;
- const int minc =
- AOMMAX(x->mv_limits.col_min * 8, ref_mv.col - MV_MAX);
- const int maxc =
- AOMMIN(x->mv_limits.col_max * 8, ref_mv.col + MV_MAX);
- const int minr =
- AOMMAX(x->mv_limits.row_min * 8, ref_mv.row - MV_MAX);
- const int maxr =
- AOMMIN(x->mv_limits.row_max * 8, ref_mv.row + MV_MAX);
-
- x->best_mv = x->second_best_mv;
- if (x->best_mv.as_mv.row * 8 <= maxr &&
- x->best_mv.as_mv.row * 8 >= minr &&
- x->best_mv.as_mv.col * 8 <= maxc &&
- x->best_mv.as_mv.col * 8 >= minc) {
- this_var = cpi->find_fractional_mv_step(
- x, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv,
- x->errorperbit, &cpi->fn_ptr[bsize],
- cpi->sf.mv.subpel_force_stop,
- cpi->sf.mv.subpel_iters_per_step,
- cond_cost_list(cpi, cost_list), x->nmvjointcost,
- x->mvcost, &distortion, &x->pred_sse[mbmi->ref_frame[0]],
- NULL,
-#if CONFIG_EXT_INTER
- NULL, 0, 0,
-#endif
- pw, ph, 1);
- if (this_var < best_mv_var) best_mv = x->best_mv.as_mv;
- x->best_mv.as_mv = best_mv;
- }
- }
-
- // Restore the reference frames.
- pd->pre[0] = backup_pred;
- } else {
- cpi->find_fractional_mv_step(
- x, &bsi->ref_mv[0]->as_mv, cm->allow_high_precision_mv,
- x->errorperbit, &cpi->fn_ptr[bsize],
- cpi->sf.mv.subpel_force_stop,
- cpi->sf.mv.subpel_iters_per_step,
- cond_cost_list(cpi, cost_list), x->nmvjointcost, x->mvcost,
- &distortion, &x->pred_sse[mbmi->ref_frame[0]], NULL,
-#if CONFIG_EXT_INTER
- NULL, 0, 0,
-#endif
- 0, 0, 0);
- }
-
-// save motion search result for use in compound prediction
-#if CONFIG_EXT_INTER
- seg_mvs[index][mbmi->ref_frame[0]].as_mv = x->best_mv.as_mv;
-#else
- seg_mvs[index][mbmi->ref_frame[0]].as_mv = x->best_mv.as_mv;
-#endif // CONFIG_EXT_INTER
- }
-
- if (cpi->sf.adaptive_motion_search)
- x->pred_mv[mbmi->ref_frame[0]] = x->best_mv.as_mv;
-
-#if CONFIG_EXT_INTER
- mode_mv[this_mode][0] = x->best_mv;
-#else
- mode_mv[NEWMV][0] = x->best_mv;
-#endif // CONFIG_EXT_INTER
-
- // restore src pointers
- mi_buf_restore(x, orig_src, orig_pre);
- }
-
- if (has_second_rf) {
-#if CONFIG_EXT_INTER
- if (seg_mvs[index][mbmi->ref_frame[1]].as_int == INVALID_MV ||
- seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV)
-#else
- if (seg_mvs[index][mbmi->ref_frame[1]].as_int == INVALID_MV ||
- seg_mvs[index][mbmi->ref_frame[0]].as_int == INVALID_MV)
-#endif // CONFIG_EXT_INTER
- continue;
- }
-
-#if CONFIG_DUAL_FILTER
- (void)run_mv_search;
-#endif // CONFIG_DUAL_FILTER
-
- if (has_second_rf &&
-#if CONFIG_EXT_INTER
- this_mode == NEW_NEWMV &&
-#else
- this_mode == NEWMV &&
-#endif // CONFIG_EXT_INTER
-#if CONFIG_DUAL_FILTER
- (mbmi->interp_filter[0] == EIGHTTAP_REGULAR || run_mv_search))
-#else
- (mbmi->interp_filter == EIGHTTAP_REGULAR || run_mv_search))
-#endif // CONFIG_DUAL_FILTER
- {
- // adjust src pointers
- mi_buf_shift(x, index);
- if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
- int rate_mv;
- frame_mv[this_mode][mbmi->ref_frame[0]].as_int =
- seg_mvs[index][mbmi->ref_frame[0]].as_int;
- frame_mv[this_mode][mbmi->ref_frame[1]].as_int =
- seg_mvs[index][mbmi->ref_frame[1]].as_int;
- joint_motion_search(cpi, x, bsize, frame_mv[this_mode], mi_row,
- mi_col,
-#if CONFIG_EXT_INTER
- bsi->ref_mv, NULL, 0,
-#endif // CONFIG_EXT_INTER
- &rate_mv, index);
-#if CONFIG_EXT_INTER
- compound_seg_newmvs[index][0].as_int =
- frame_mv[this_mode][mbmi->ref_frame[0]].as_int;
- compound_seg_newmvs[index][1].as_int =
- frame_mv[this_mode][mbmi->ref_frame[1]].as_int;
-#else
- seg_mvs[index][mbmi->ref_frame[0]].as_int =
- frame_mv[this_mode][mbmi->ref_frame[0]].as_int;
- seg_mvs[index][mbmi->ref_frame[1]].as_int =
- frame_mv[this_mode][mbmi->ref_frame[1]].as_int;
-#endif // CONFIG_EXT_INTER
- }
- // restore src pointers
- mi_buf_restore(x, orig_src, orig_pre);
- }
-
- bsi->rdstat[index][mode_idx].brate = set_and_cost_bmi_mvs(
- cpi, x, xd, index, this_mode, mode_mv[this_mode], frame_mv,
- seg_mvs[index],
-#if CONFIG_EXT_INTER
- compound_seg_newmvs[index],
-#endif // CONFIG_EXT_INTER
- bsi->ref_mv, x->nmvjointcost, x->mvcost, mi_row, mi_col);
-
- for (ref = 0; ref < 1 + has_second_rf; ++ref) {
- bsi->rdstat[index][mode_idx].mvs[ref].as_int =
- mode_mv[this_mode][ref].as_int;
- if (num_4x4_blocks_wide > 1)
- bsi->rdstat[index + 1][mode_idx].mvs[ref].as_int =
- mode_mv[this_mode][ref].as_int;
- if (num_4x4_blocks_high > 1)
- bsi->rdstat[index + 2][mode_idx].mvs[ref].as_int =
- mode_mv[this_mode][ref].as_int;
- bsi->rdstat[index][mode_idx].pred_mv[ref].as_int =
- mi->bmi[index].pred_mv[ref].as_int;
- if (num_4x4_blocks_wide > 1)
- bsi->rdstat[index + 1][mode_idx].pred_mv[ref].as_int =
- mi->bmi[index].pred_mv[ref].as_int;
- if (num_4x4_blocks_high > 1)
- bsi->rdstat[index + 2][mode_idx].pred_mv[ref].as_int =
- mi->bmi[index].pred_mv[ref].as_int;
-#if CONFIG_EXT_INTER
- bsi->rdstat[index][mode_idx].ref_mv[ref].as_int =
- bsi->ref_mv[ref]->as_int;
- if (num_4x4_blocks_wide > 1)
- bsi->rdstat[index + 1][mode_idx].ref_mv[ref].as_int =
- bsi->ref_mv[ref]->as_int;
- if (num_4x4_blocks_high > 1)
- bsi->rdstat[index + 2][mode_idx].ref_mv[ref].as_int =
- bsi->ref_mv[ref]->as_int;
-#endif // CONFIG_EXT_INTER
- }
-
- // Trap vectors that reach beyond the UMV borders
- if (mv_check_bounds(&x->mv_limits, &mode_mv[this_mode][0].as_mv) ||
- (has_second_rf &&
- mv_check_bounds(&x->mv_limits, &mode_mv[this_mode][1].as_mv)))
- continue;
-
- if (filter_idx > 0) {
- BEST_SEG_INFO *ref_bsi = bsi_buf;
- subpelmv = 0;
- have_ref = 1;
-
- for (ref = 0; ref < 1 + has_second_rf; ++ref) {
- subpelmv |= mv_has_subpel(&mode_mv[this_mode][ref].as_mv);
-#if CONFIG_EXT_INTER
- if (have_newmv_in_inter_mode(this_mode))
- have_ref &=
- ((mode_mv[this_mode][ref].as_int ==
- ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int) &&
- (bsi->ref_mv[ref]->as_int ==
- ref_bsi->rdstat[index][mode_idx].ref_mv[ref].as_int));
- else
-#endif // CONFIG_EXT_INTER
- have_ref &= mode_mv[this_mode][ref].as_int ==
- ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int;
- }
-
- have_ref &= ref_bsi->rdstat[index][mode_idx].brate > 0;
-
- if (filter_idx > 1 && !subpelmv && !have_ref) {
- ref_bsi = bsi_buf + 1;
- have_ref = 1;
- for (ref = 0; ref < 1 + has_second_rf; ++ref)
-#if CONFIG_EXT_INTER
- if (have_newmv_in_inter_mode(this_mode))
- have_ref &=
- ((mode_mv[this_mode][ref].as_int ==
- ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int) &&
- (bsi->ref_mv[ref]->as_int ==
- ref_bsi->rdstat[index][mode_idx].ref_mv[ref].as_int));
- else
-#endif // CONFIG_EXT_INTER
- have_ref &= mode_mv[this_mode][ref].as_int ==
- ref_bsi->rdstat[index][mode_idx].mvs[ref].as_int;
-
- have_ref &= ref_bsi->rdstat[index][mode_idx].brate > 0;
- }
-
- if (!subpelmv && have_ref &&
- ref_bsi->rdstat[index][mode_idx].brdcost < INT64_MAX) {
- bsi->rdstat[index][mode_idx].byrate =
- ref_bsi->rdstat[index][mode_idx].byrate;
- bsi->rdstat[index][mode_idx].bdist =
- ref_bsi->rdstat[index][mode_idx].bdist;
- bsi->rdstat[index][mode_idx].bsse =
- ref_bsi->rdstat[index][mode_idx].bsse;
- bsi->rdstat[index][mode_idx].brate +=
- ref_bsi->rdstat[index][mode_idx].byrate;
- bsi->rdstat[index][mode_idx].eobs =
- ref_bsi->rdstat[index][mode_idx].eobs;
-
- bsi->rdstat[index][mode_idx].brdcost =
- RDCOST(x->rdmult, x->rddiv, bsi->rdstat[index][mode_idx].brate,
- bsi->rdstat[index][mode_idx].bdist);
-
- memcpy(bsi->rdstat[index][mode_idx].ta,
- ref_bsi->rdstat[index][mode_idx].ta,
- sizeof(bsi->rdstat[index][mode_idx].ta));
- memcpy(bsi->rdstat[index][mode_idx].tl,
- ref_bsi->rdstat[index][mode_idx].tl,
- sizeof(bsi->rdstat[index][mode_idx].tl));
- if (num_4x4_blocks_wide > 1)
- bsi->rdstat[index + 1][mode_idx].eobs =
- ref_bsi->rdstat[index + 1][mode_idx].eobs;
- if (num_4x4_blocks_high > 1)
- bsi->rdstat[index + 2][mode_idx].eobs =
- ref_bsi->rdstat[index + 2][mode_idx].eobs;
-
- if (bsi->rdstat[index][mode_idx].brdcost < new_best_rd) {
- // If the NEWMV mode is using the same motion vector as the
- // NEARESTMV mode, skip the rest rate-distortion calculations
- // and use the inferred motion vector modes.
- if (this_mode == NEWMV) {
- if (has_second_rf) {
- if (bsi->rdstat[index][mode_idx].mvs[0].as_int ==
- bsi->ref_mv[0]->as_int &&
- bsi->rdstat[index][mode_idx].mvs[1].as_int ==
- bsi->ref_mv[1]->as_int)
- continue;
- } else {
- if (bsi->rdstat[index][mode_idx].mvs[0].as_int ==
- bsi->ref_mv[0]->as_int)
- continue;
- }
- }
- mode_selected = this_mode;
- new_best_rd = bsi->rdstat[index][mode_idx].brdcost;
-#if CONFIG_PVQ
- od_encode_checkpoint(&x->daala_enc, &post_buf);
-#endif // CONFIG_PVQ
- }
- continue;
- }
- }
-
- bsi->rdstat[index][mode_idx].brdcost = encode_inter_mb_segment_sub8x8(
- cpi, x, bsi->segment_rd - this_segment_rd, index,
- &bsi->rdstat[index][mode_idx].byrate,
- &bsi->rdstat[index][mode_idx].bdist,
- &bsi->rdstat[index][mode_idx].bsse, bsi->rdstat[index][mode_idx].ta,
- bsi->rdstat[index][mode_idx].tl, idy, idx, mi_row, mi_col);
-
- if (bsi->rdstat[index][mode_idx].brdcost < INT64_MAX) {
- bsi->rdstat[index][mode_idx].brdcost += RDCOST(
- x->rdmult, x->rddiv, bsi->rdstat[index][mode_idx].brate, 0);
- bsi->rdstat[index][mode_idx].brate +=
- bsi->rdstat[index][mode_idx].byrate;
- bsi->rdstat[index][mode_idx].eobs = p->eobs[index];
- if (num_4x4_blocks_wide > 1)
- bsi->rdstat[index + 1][mode_idx].eobs = p->eobs[index + 1];
- if (num_4x4_blocks_high > 1)
- bsi->rdstat[index + 2][mode_idx].eobs = p->eobs[index + 2];
- }
-
- if (bsi->rdstat[index][mode_idx].brdcost < new_best_rd) {
- // If the NEWMV mode is using the same motion vector as the
- // NEARESTMV mode, skip the rest rate-distortion calculations
- // and use the inferred motion vector modes.
- if (this_mode == NEWMV) {
- if (has_second_rf) {
- if (bsi->rdstat[index][mode_idx].mvs[0].as_int ==
- bsi->ref_mv[0]->as_int &&
- bsi->rdstat[index][mode_idx].mvs[1].as_int ==
- bsi->ref_mv[1]->as_int)
- continue;
- } else {
- if (bsi->rdstat[index][mode_idx].mvs[0].as_int ==
- bsi->ref_mv[0]->as_int)
- continue;
- }
- }
- mode_selected = this_mode;
- new_best_rd = bsi->rdstat[index][mode_idx].brdcost;
-
-#if CONFIG_PVQ
- od_encode_checkpoint(&x->daala_enc, &post_buf);
-#endif // CONFIG_PVQ
- }
- } /*for each 4x4 mode*/
-
- if (new_best_rd == INT64_MAX) {
- int iy, midx;
- for (iy = index + 1; iy < 4; ++iy)
-#if CONFIG_EXT_INTER
- for (midx = 0; midx < INTER_MODES + INTER_COMPOUND_MODES; ++midx)
-#else
- for (midx = 0; midx < INTER_MODES; ++midx)
-#endif // CONFIG_EXT_INTER
- bsi->rdstat[iy][midx].brdcost = INT64_MAX;
- bsi->segment_rd = INT64_MAX;
-#if CONFIG_PVQ
- od_encode_rollback(&x->daala_enc, &pre_buf);
-#endif // CONFIG_PVQ
- return INT64_MAX;
- }
-
- mode_idx = INTER_OFFSET(mode_selected);
- memcpy(t_above, bsi->rdstat[index][mode_idx].ta, sizeof(t_above));
- memcpy(t_left, bsi->rdstat[index][mode_idx].tl, sizeof(t_left));
-#if CONFIG_PVQ
- od_encode_rollback(&x->daala_enc, &post_buf);
-#endif // CONFIG_PVQ
-
-#if CONFIG_EXT_INTER
- bsi->ref_mv[0]->as_int = bsi->rdstat[index][mode_idx].ref_mv[0].as_int;
- if (has_second_rf)
- bsi->ref_mv[1]->as_int = bsi->rdstat[index][mode_idx].ref_mv[1].as_int;
-#endif // CONFIG_EXT_INTER
- set_and_cost_bmi_mvs(cpi, x, xd, index, mode_selected,
- mode_mv[mode_selected], frame_mv, seg_mvs[index],
-#if CONFIG_EXT_INTER
- compound_seg_newmvs[index],
-#endif // CONFIG_EXT_INTER
- bsi->ref_mv, x->nmvjointcost, x->mvcost, mi_row,
- mi_col);
-
- br += bsi->rdstat[index][mode_idx].brate;
- bd += bsi->rdstat[index][mode_idx].bdist;
- block_sse += bsi->rdstat[index][mode_idx].bsse;
- segmentyrate += bsi->rdstat[index][mode_idx].byrate;
- this_segment_rd += bsi->rdstat[index][mode_idx].brdcost;
-
- if (this_segment_rd > bsi->segment_rd) {
- int iy, midx;
- for (iy = index + 1; iy < 4; ++iy)
-#if CONFIG_EXT_INTER
- for (midx = 0; midx < INTER_MODES + INTER_COMPOUND_MODES; ++midx)
-#else
- for (midx = 0; midx < INTER_MODES; ++midx)
-#endif // CONFIG_EXT_INTER
- bsi->rdstat[iy][midx].brdcost = INT64_MAX;
- bsi->segment_rd = INT64_MAX;
-#if CONFIG_PVQ
- od_encode_rollback(&x->daala_enc, &pre_buf);
-#endif // CONFIG_PVQ
- return INT64_MAX;
- }
- }
- } /* for each label */
-#if CONFIG_PVQ
- od_encode_rollback(&x->daala_enc, &pre_buf);
-#endif // CONFIG_PVQ
-
- bsi->r = br;
- bsi->d = bd;
- bsi->segment_yrate = segmentyrate;
- bsi->segment_rd = this_segment_rd;
- bsi->sse = block_sse;
-
- // update the coding decisions
- for (k = 0; k < 4; ++k) bsi->modes[k] = mi->bmi[k].as_mode;
-
-#if CONFIG_DAALA_DIST
- // Compute prediction (i.e. skip) and decoded distortion by daala-distortion.
- {
- const int src_stride = p->src.stride;
- const int dst_stride = pd->dst.stride;
- uint8_t *src = p->src.buf;
- uint8_t *dst = pd->dst.buf;
- const BLOCK_SIZE plane_bsize = get_plane_block_size(mi->mbmi.sb_type, pd);
- const int use_activity_masking = 0;
- const int qm = OD_HVS_QM;
- const int bsw = block_size_wide[plane_bsize];
- const int bsh = block_size_high[plane_bsize];
- int64_t rd1, rd2;
- int64_t daala_sse, daala_dist;
- TX_SIZE tx_size = mbmi->tx_size;
-
-#if CONFIG_HIGHBITDEPTH
- uint8_t *recon_8x8;
- DECLARE_ALIGNED(16, uint16_t, recon16[8 * 8]);
-
- if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
- recon_8x8 = CONVERT_TO_BYTEPTR(recon16);
- else
- recon_8x8 = (uint8_t *)recon16;
-#else
- DECLARE_ALIGNED(16, uint8_t, recon_8x8[8 * 8]);
-#endif // CONFIG_HIGHBITDEPTH
-
-#if CONFIG_PVQ
- use_activity_masking = x->daala_enc.use_activity_masking;
-#endif // CONFIG_PVQ
-
- // For each of sub8x8 prediction block in a 8x8 block
- for (idy = 0; idy < 2; idy += num_4x4_blocks_high) {
- for (idx = 0; idx < 2; idx += num_4x4_blocks_wide) {
- int i = idy * 2 + idx;
- const uint8_t *const src_sub8x8 =
- src + av1_raster_block_offset(BLOCK_8X8, i, p->src.stride);
- uint8_t *const dst_sub8x8 =
- dst + av1_raster_block_offset(BLOCK_8X8, i, pd->dst.stride);
- uint8_t *recon_sub8x8 = recon_8x8 + (idy * 8 + idx) * 4;
- const int txb_width = max_block_wide(xd, plane_bsize, 0);
- const int txb_height = max_block_high(xd, plane_bsize, 0);
- int idx_, idy_;
-
- av1_build_inter_predictor_sub8x8(cm, xd, 0, i, idy, idx, mi_row,
- mi_col);
-#if CONFIG_HIGHBITDEPTH
- if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
- aom_highbd_subtract_block(
- bsh, bsw,
- av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), 8,
- src_sub8x8, p->src.stride, dst_sub8x8, pd->dst.stride, xd->bd);
- } else {
- aom_subtract_block(
- bsh, bsw,
- av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff), 8,
- src_sub8x8, p->src.stride, dst_sub8x8, pd->dst.stride);
- }
-#else
- aom_subtract_block(
- bsh, bsw, av1_raster_block_offset_int16(BLOCK_8X8, i, p->src_diff),
- 8, src_sub8x8, p->src.stride, dst_sub8x8, pd->dst.stride);
-#endif // CONFIG_HIGHBITDEPTH
-
-#if CONFIG_HIGHBITDEPTH
- if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
- aom_highbd_convolve_copy(dst_sub8x8, dst_stride, recon_sub8x8, 8,
- NULL, 0, NULL, 0, bsw, bsh, xd->bd);
- } else {
-#endif // CONFIG_HIGHBITDEPTH
- aom_convolve_copy(dst_sub8x8, dst_stride, recon_sub8x8, 8, NULL, 0,
- NULL, 0, bsw, bsh);
-#if CONFIG_HIGHBITDEPTH
- }
-#endif // CONFIG_HIGHBITDEPTH
-
- // To get decoded pixels, do 4x4 xform and quant for each 4x4 block
- // in a sub8x8 prediction block. In case remaining parts of
- // sub8x8 inter mode rdo assume pd->dst stores predicted pixels,
- // use local buffer to store decoded pixels.
- for (idy_ = 0; idy_ < txb_height; idy_++) {
- for (idx_ = 0; idx_ < txb_width; idx_++) {
- int coeff_ctx = 0;
- const tran_low_t *dqcoeff;
- uint16_t eob;
- const PLANE_TYPE plane_type = PLANE_TYPE_Y;
- uint8_t *recon_4x4 = recon_sub8x8 + (idy_ * 8 + idx_) * 4;
- const int block_raster_idx = (idy + idy_) * 2 + (idx + idx_);
- const int block =
- av1_raster_order_to_block_index(tx_size, block_raster_idx);
- TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
-
- dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
- av1_xform_quant(cm, x, 0, block, idy + idy_, idx + idx_, BLOCK_8X8,
- tx_size, coeff_ctx, AV1_XFORM_QUANT_FP);
- av1_optimize_b(cm, x, 0, block, tx_size, coeff_ctx);
-
- eob = p->eobs[block];
- av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size,
- recon_4x4, 8, eob);
- }
- }
- }
- }
- // Compute daala-distortion for a 8x8 block
- daala_sse = av1_daala_dist(src, src_stride, pd->dst.buf, dst_stride, 8, 8,
- qm, use_activity_masking, x->qindex)
- << 4;
-
- daala_dist = av1_daala_dist(src, src_stride, recon_8x8, 8, 8, 8, qm,
- use_activity_masking, x->qindex)
- << 4;
-
- bsi->sse = daala_sse;
- bsi->d = daala_dist;
-
- rd1 = RDCOST(x->rdmult, x->rddiv, bsi->r, bsi->d);
- rd2 = RDCOST(x->rdmult, x->rddiv, 0, bsi->sse);
- bsi->segment_rd = AOMMIN(rd1, rd2);
- }
-#endif // CONFIG_DAALA_DIST
-
- if (bsi->segment_rd > best_rd) return INT64_MAX;
- /* set it to the best */
- for (idx = 0; idx < 4; idx++) {
- mode_idx = INTER_OFFSET(bsi->modes[idx]);
- mi->bmi[idx].as_mv[0].as_int = bsi->rdstat[idx][mode_idx].mvs[0].as_int;
- if (has_second_ref(mbmi))
- mi->bmi[idx].as_mv[1].as_int = bsi->rdstat[idx][mode_idx].mvs[1].as_int;
- mi->bmi[idx].pred_mv[0] = bsi->rdstat[idx][mode_idx].pred_mv[0];
- if (has_second_ref(mbmi))
- mi->bmi[idx].pred_mv[1] = bsi->rdstat[idx][mode_idx].pred_mv[1];
-#if CONFIG_EXT_INTER
- mi->bmi[idx].ref_mv[0].as_int = bsi->rdstat[idx][mode_idx].ref_mv[0].as_int;
- if (has_second_rf)
- mi->bmi[idx].ref_mv[1].as_int =
- bsi->rdstat[idx][mode_idx].ref_mv[1].as_int;
-#endif // CONFIG_EXT_INTER
- x->plane[0].eobs[idx] = bsi->rdstat[idx][mode_idx].eobs;
- mi->bmi[idx].as_mode = bsi->modes[idx];
- }
-
- /*
- * used to set mbmi->mv.as_int
- */
- *returntotrate = bsi->r;
- *returndistortion = bsi->d;
- *returnyrate = bsi->segment_yrate;
- *skippable = av1_is_skippable_in_plane(x, BLOCK_8X8, 0);
- *psse = bsi->sse;
- mbmi->mode = bsi->modes[3];
-
- return bsi->segment_rd;
-}
-
static void estimate_ref_frame_costs(const AV1_COMMON *cm,
const MACROBLOCKD *xd, int segment_id,
unsigned int *ref_costs_single,
@@ -11736,787 +10517,6 @@
store_coding_context(x, ctx, THR_ZEROMV, best_pred_diff, 0);
}
-void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi,
- TileDataEnc *tile_data, struct macroblock *x,
- int mi_row, int mi_col,
- struct RD_STATS *rd_cost,
-#if CONFIG_SUPERTX
- int *returnrate_nocoef,
-#endif // CONFIG_SUPERTX
- BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx,
- int64_t best_rd_so_far) {
- const AV1_COMMON *const cm = &cpi->common;
- const RD_OPT *const rd_opt = &cpi->rd;
- const SPEED_FEATURES *const sf = &cpi->sf;
- MACROBLOCKD *const xd = &x->e_mbd;
- MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
- const struct segmentation *const seg = &cm->seg;
- MV_REFERENCE_FRAME ref_frame, second_ref_frame;
- unsigned char segment_id = mbmi->segment_id;
- int comp_pred, i;
- int_mv frame_mv[MB_MODE_COUNT][TOTAL_REFS_PER_FRAME];
- struct buf_2d yv12_mb[TOTAL_REFS_PER_FRAME][MAX_MB_PLANE];
- static const int flag_list[TOTAL_REFS_PER_FRAME] = {
- 0,
- AOM_LAST_FLAG,
-#if CONFIG_EXT_REFS
- AOM_LAST2_FLAG,
- AOM_LAST3_FLAG,
-#endif // CONFIG_EXT_REFS
- AOM_GOLD_FLAG,
-#if CONFIG_EXT_REFS
- AOM_BWD_FLAG,
-#endif // CONFIG_EXT_REFS
- AOM_ALT_FLAG
- };
- int64_t best_rd = best_rd_so_far;
- int64_t best_yrd = best_rd_so_far; // FIXME(rbultje) more precise
- int64_t best_pred_diff[REFERENCE_MODES];
- int64_t best_pred_rd[REFERENCE_MODES];
- MB_MODE_INFO best_mbmode;
- int ref_index, best_ref_index = 0;
- unsigned int ref_costs_single[TOTAL_REFS_PER_FRAME];
- unsigned int ref_costs_comp[TOTAL_REFS_PER_FRAME];
- aom_prob comp_mode_p;
-#if CONFIG_DUAL_FILTER
- InterpFilter tmp_best_filter[4] = { 0 };
-#else
- InterpFilter tmp_best_filter = SWITCHABLE;
-#endif // CONFIG_DUAL_FILTER
- int rate_uv_intra, rate_uv_tokenonly = INT_MAX;
- int64_t dist_uv = INT64_MAX;
- int skip_uv;
- PREDICTION_MODE mode_uv = DC_PRED;
- const int intra_cost_penalty = av1_get_intra_cost_penalty(
- cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth);
- int_mv seg_mvs[4][TOTAL_REFS_PER_FRAME];
- b_mode_info best_bmodes[4];
- int best_skip2 = 0;
- int ref_frame_skip_mask[2] = { 0 };
- int internal_active_edge =
- av1_active_edge_sb(cpi, mi_row, mi_col) && av1_internal_image_edge(cpi);
-#if CONFIG_PVQ
- od_rollback_buffer pre_buf;
-
- od_encode_checkpoint(&x->daala_enc, &pre_buf);
-#endif // CONFIG_PVQ
-
-#if CONFIG_SUPERTX
- best_rd_so_far = INT64_MAX;
- best_rd = best_rd_so_far;
- best_yrd = best_rd_so_far;
-#endif // CONFIG_SUPERTX
- av1_zero(best_mbmode);
-
-#if CONFIG_FILTER_INTRA
- mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 0;
- mbmi->filter_intra_mode_info.use_filter_intra_mode[1] = 0;
-#endif // CONFIG_FILTER_INTRA
- mbmi->motion_mode = SIMPLE_TRANSLATION;
-#if CONFIG_EXT_INTER
- mbmi->interinter_compound_type = COMPOUND_AVERAGE;
- mbmi->use_wedge_interintra = 0;
-#endif // CONFIG_EXT_INTER
-#if CONFIG_WARPED_MOTION
- mbmi->num_proj_ref[0] = 0;
- mbmi->num_proj_ref[1] = 0;
-#endif // CONFIG_WARPED_MOTION
-
- for (i = 0; i < 4; i++) {
- int j;
- for (j = 0; j < TOTAL_REFS_PER_FRAME; j++)
- seg_mvs[i][j].as_int = INVALID_MV;
- }
-
- estimate_ref_frame_costs(cm, xd, segment_id, ref_costs_single, ref_costs_comp,
- &comp_mode_p);
-
- for (i = 0; i < REFERENCE_MODES; ++i) best_pred_rd[i] = INT64_MAX;
- rate_uv_intra = INT_MAX;
-
- rd_cost->rate = INT_MAX;
-#if CONFIG_SUPERTX
- *returnrate_nocoef = INT_MAX;
-#endif // CONFIG_SUPERTX
-
- for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
- x->mbmi_ext->mode_context[ref_frame] = 0;
-#if CONFIG_EXT_INTER
- x->mbmi_ext->compound_mode_context[ref_frame] = 0;
-#endif // CONFIG_EXT_INTER
- if (cpi->ref_frame_flags & flag_list[ref_frame]) {
- setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col,
- frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb);
- } else {
- ref_frame_skip_mask[0] |= (1 << ref_frame);
- ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
- }
- frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
-#if CONFIG_EXT_INTER
-#endif // CONFIG_EXT_INTER
- frame_mv[ZEROMV][ref_frame].as_int = 0;
- }
-
-#if CONFIG_PALETTE
- mbmi->palette_mode_info.palette_size[0] = 0;
- mbmi->palette_mode_info.palette_size[1] = 0;
-#endif // CONFIG_PALETTE
-
- for (ref_index = 0; ref_index < MAX_REFS; ++ref_index) {
- int mode_excluded = 0;
- int64_t this_rd = INT64_MAX;
- int disable_skip = 0;
- int compmode_cost = 0;
- int rate2 = 0, rate_y = 0, rate_uv = 0;
- int64_t distortion2 = 0, distortion_y = 0, distortion_uv = 0;
- int skippable = 0;
- int this_skip2 = 0;
- int64_t total_sse = INT_MAX;
-
-#if CONFIG_PVQ
- od_encode_rollback(&x->daala_enc, &pre_buf);
-#endif // CONFIG_PVQ
-
- ref_frame = av1_ref_order[ref_index].ref_frame[0];
- second_ref_frame = av1_ref_order[ref_index].ref_frame[1];
-
- mbmi->ref_mv_idx = 0;
-
- // Look at the reference frame of the best mode so far and set the
- // skip mask to look at a subset of the remaining modes.
- if (ref_index > 2 && sf->mode_skip_start < MAX_MODES) {
- if (ref_index == 3) {
- switch (best_mbmode.ref_frame[0]) {
- case INTRA_FRAME: break;
- case LAST_FRAME:
- ref_frame_skip_mask[0] |= (1 << GOLDEN_FRAME) |
-#if CONFIG_EXT_REFS
- (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |
- (1 << BWDREF_FRAME) |
-#endif // CONFIG_EXT_REFS
- (1 << ALTREF_FRAME);
- ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
- break;
-#if CONFIG_EXT_REFS
- case LAST2_FRAME:
- ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << LAST3_FRAME) |
- (1 << GOLDEN_FRAME) |
- (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME);
- ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
- break;
- case LAST3_FRAME:
- ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << LAST2_FRAME) |
- (1 << GOLDEN_FRAME) |
- (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME);
- ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
- break;
-#endif // CONFIG_EXT_REFS
- case GOLDEN_FRAME:
- ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
-#if CONFIG_EXT_REFS
- (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |
- (1 << BWDREF_FRAME) |
-#endif // CONFIG_EXT_REFS
- (1 << ALTREF_FRAME);
- ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK;
- break;
-#if CONFIG_EXT_REFS
- case BWDREF_FRAME:
- ref_frame_skip_mask[0] |= (1 << LAST_FRAME) | (1 << LAST2_FRAME) |
- (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) |
- (1 << ALTREF_FRAME);
- ref_frame_skip_mask[1] |= (1 << ALTREF_FRAME) | 0x01;
- break;
-#endif // CONFIG_EXT_REFS
- case ALTREF_FRAME:
- ref_frame_skip_mask[0] |= (1 << LAST_FRAME) |
-#if CONFIG_EXT_REFS
- (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |
- (1 << BWDREF_FRAME) |
-#endif // CONFIG_EXT_REFS
- (1 << GOLDEN_FRAME);
-#if CONFIG_EXT_REFS
- ref_frame_skip_mask[1] |= (1 << BWDREF_FRAME) | 0x01;
-#endif // CONFIG_EXT_REFS
- break;
- case NONE_FRAME:
- case TOTAL_REFS_PER_FRAME:
- assert(0 && "Invalid Reference frame");
- break;
- }
- }
- }
-
- if ((ref_frame_skip_mask[0] & (1 << ref_frame)) &&
- (ref_frame_skip_mask[1] & (1 << AOMMAX(0, second_ref_frame))))
- continue;
-
- // Test best rd so far against threshold for trying this mode.
- if (!internal_active_edge &&
- rd_less_than_thresh(best_rd,
- rd_opt->threshes[segment_id][bsize][ref_index],
- tile_data->thresh_freq_fact[bsize][ref_index]))
- continue;
-
- // This is only used in motion vector unit test.
- if (cpi->oxcf.motion_vector_unit_test && ref_frame == INTRA_FRAME) continue;
-
-#if CONFIG_ONE_SIDED_COMPOUND // Changes LL bitstream
-#if CONFIG_EXT_REFS
- if (cpi->oxcf.pass == 0) {
- // Complexity-compression trade-offs
- // if (ref_frame == ALTREF_FRAME) continue;
- // if (ref_frame == BWDREF_FRAME) continue;
- if (second_ref_frame == ALTREF_FRAME) continue;
- // if (second_ref_frame == BWDREF_FRAME) continue;
- }
-#endif
-#endif
- comp_pred = second_ref_frame > INTRA_FRAME;
- if (comp_pred) {
- if (!cpi->allow_comp_inter_inter) continue;
- if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue;
- // Do not allow compound prediction if the segment level reference frame
- // feature is in use as in this case there can only be one reference.
- if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue;
-
- if ((sf->mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) &&
- best_mbmode.ref_frame[0] == INTRA_FRAME)
- continue;
- }
-
- // TODO(jingning, jkoleszar): scaling reference frame not supported for
- // sub8x8 blocks.
- if (ref_frame > INTRA_FRAME &&
- av1_is_scaled(&cm->frame_refs[ref_frame - 1].sf))
- continue;
-
- if (second_ref_frame > INTRA_FRAME &&
- av1_is_scaled(&cm->frame_refs[second_ref_frame - 1].sf))
- continue;
-
- if (comp_pred)
- mode_excluded = cm->reference_mode == SINGLE_REFERENCE;
- else if (ref_frame != INTRA_FRAME)
- mode_excluded = cm->reference_mode == COMPOUND_REFERENCE;
-
- // If the segment reference frame feature is enabled....
- // then do nothing if the current ref frame is not allowed..
- if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME) &&
- get_segdata(seg, segment_id, SEG_LVL_REF_FRAME) != (int)ref_frame) {
- continue;
- // Disable this drop out case if the ref frame
- // segment level feature is enabled for this segment. This is to
- // prevent the possibility that we end up unable to pick any mode.
- } else if (!segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) {
- // Only consider ZEROMV/ALTREF_FRAME for alt ref frame,
- // unless ARNR filtering is enabled in which case we want
- // an unfiltered alternative. We allow near/nearest as well
- // because they may result in zero-zero MVs but be cheaper.
- if (cpi->rc.is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0))
- continue;
- }
-
- mbmi->tx_size = TX_4X4;
- mbmi->uv_mode = DC_PRED;
- mbmi->ref_frame[0] = ref_frame;
- mbmi->ref_frame[1] = second_ref_frame;
-// Evaluate all sub-pel filters irrespective of whether we can use
-// them for this frame.
-#if CONFIG_DUAL_FILTER
- for (i = 0; i < 4; ++i)
- mbmi->interp_filter[i] = cm->interp_filter == SWITCHABLE
- ? EIGHTTAP_REGULAR
- : cm->interp_filter;
-#else
- mbmi->interp_filter =
- cm->interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR : cm->interp_filter;
-#endif // CONFIG_DUAL_FILTER
- x->skip = 0;
- set_ref_ptrs(cm, xd, ref_frame, second_ref_frame);
-
- // Select prediction reference frames.
- for (i = 0; i < MAX_MB_PLANE; i++) {
- xd->plane[i].pre[0] = yv12_mb[ref_frame][i];
- if (comp_pred) xd->plane[i].pre[1] = yv12_mb[second_ref_frame][i];
- }
-
-#if CONFIG_VAR_TX
- mbmi->inter_tx_size[0][0] = mbmi->tx_size;
- mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
-#endif // CONFIG_VAR_TX
-
- if (ref_frame == INTRA_FRAME) {
- int rate;
- if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y, &distortion_y,
- NULL, best_rd) >= best_rd)
- continue;
- rate2 += rate;
- rate2 += intra_cost_penalty;
- distortion2 += distortion_y;
-
- if (rate_uv_intra == INT_MAX) {
- choose_intra_uv_mode(cpi, x, ctx, bsize, TX_4X4, &rate_uv_intra,
- &rate_uv_tokenonly, &dist_uv, &skip_uv, &mode_uv);
- }
- rate2 += rate_uv_intra;
- rate_uv = rate_uv_tokenonly;
- distortion2 += dist_uv;
- distortion_uv = dist_uv;
- mbmi->uv_mode = mode_uv;
- } else {
- int rate;
- int64_t distortion;
- int64_t this_rd_thresh;
- int64_t tmp_rd, tmp_best_rd = INT64_MAX, tmp_best_rdu = INT64_MAX;
- int tmp_best_rate = INT_MAX, tmp_best_ratey = INT_MAX;
- int64_t tmp_best_distortion = INT_MAX, tmp_best_sse, uv_sse;
- int tmp_best_skippable = 0;
- int switchable_filter_index;
- int_mv *second_ref =
- comp_pred ? &x->mbmi_ext->ref_mvs[second_ref_frame][0] : NULL;
- b_mode_info tmp_best_bmodes[16]; // Should this be 4 ?
- MB_MODE_INFO tmp_best_mbmode;
-#if CONFIG_DUAL_FILTER
- BEST_SEG_INFO bsi[DUAL_FILTER_SET_SIZE];
-#else
- BEST_SEG_INFO bsi[SWITCHABLE_FILTERS];
-#endif // CONFIG_DUAL_FILTER
- int pred_exists = 0;
- int uv_skippable;
-#if CONFIG_EXT_INTER
- int_mv compound_seg_newmvs[4][2];
- for (i = 0; i < 4; i++) {
- compound_seg_newmvs[i][0].as_int = INVALID_MV;
- compound_seg_newmvs[i][1].as_int = INVALID_MV;
- }
-#endif // CONFIG_EXT_INTER
-
- this_rd_thresh = (ref_frame == LAST_FRAME)
- ? rd_opt->threshes[segment_id][bsize][THR_LAST]
- : rd_opt->threshes[segment_id][bsize][THR_ALTR];
-#if CONFIG_EXT_REFS
- this_rd_thresh = (ref_frame == LAST2_FRAME)
- ? rd_opt->threshes[segment_id][bsize][THR_LAST2]
- : this_rd_thresh;
- this_rd_thresh = (ref_frame == LAST3_FRAME)
- ? rd_opt->threshes[segment_id][bsize][THR_LAST3]
- : this_rd_thresh;
- this_rd_thresh = (ref_frame == BWDREF_FRAME)
- ? rd_opt->threshes[segment_id][bsize][THR_BWDR]
- : this_rd_thresh;
-#endif // CONFIG_EXT_REFS
- this_rd_thresh = (ref_frame == GOLDEN_FRAME)
- ? rd_opt->threshes[segment_id][bsize][THR_GOLD]
- : this_rd_thresh;
-
- // TODO(any): Add search of the tx_type to improve rd performance at the
- // expense of speed.
- mbmi->tx_type = DCT_DCT;
-
- if (cm->interp_filter != BILINEAR) {
-#if CONFIG_DUAL_FILTER
- tmp_best_filter[0] = EIGHTTAP_REGULAR;
- tmp_best_filter[1] = EIGHTTAP_REGULAR;
- tmp_best_filter[2] = EIGHTTAP_REGULAR;
- tmp_best_filter[3] = EIGHTTAP_REGULAR;
-#else
- tmp_best_filter = EIGHTTAP_REGULAR;
-#endif // CONFIG_DUAL_FILTER
- if (x->source_variance < sf->disable_filter_search_var_thresh) {
-#if CONFIG_DUAL_FILTER
- tmp_best_filter[0] = EIGHTTAP_REGULAR;
-#else
- tmp_best_filter = EIGHTTAP_REGULAR;
-#endif // CONFIG_DUAL_FILTER
- } else if (sf->adaptive_pred_interp_filter == 1 &&
- ctx->pred_interp_filter < SWITCHABLE) {
-#if CONFIG_DUAL_FILTER
- tmp_best_filter[0] = ctx->pred_interp_filter;
-#else
- tmp_best_filter = ctx->pred_interp_filter;
-#endif // CONFIG_DUAL_FILTER
- } else if (sf->adaptive_pred_interp_filter == 2) {
-#if CONFIG_DUAL_FILTER
- tmp_best_filter[0] = ctx->pred_interp_filter < SWITCHABLE
- ? ctx->pred_interp_filter
- : 0;
-#else
- tmp_best_filter = ctx->pred_interp_filter < SWITCHABLE
- ? ctx->pred_interp_filter
- : 0;
-#endif // CONFIG_DUAL_FILTER
- } else {
-#if CONFIG_DUAL_FILTER
- const int filter_set_size = DUAL_FILTER_SET_SIZE;
-#else
- const int filter_set_size = SWITCHABLE_FILTERS;
-#endif // CONFIG_DUAL_FILTER
- for (switchable_filter_index = 0;
- switchable_filter_index < filter_set_size;
- ++switchable_filter_index) {
- int newbest, rs;
- int64_t rs_rd;
- MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
-#if CONFIG_DUAL_FILTER
- mbmi->interp_filter[0] = filter_sets[switchable_filter_index][0];
- mbmi->interp_filter[1] = filter_sets[switchable_filter_index][1];
- mbmi->interp_filter[2] = filter_sets[switchable_filter_index][0];
- mbmi->interp_filter[3] = filter_sets[switchable_filter_index][1];
-#else
- mbmi->interp_filter = switchable_filter_index;
-#endif // CONFIG_DUAL_FILTER
- tmp_rd = rd_pick_inter_best_sub8x8_mode(
- cpi, x, &mbmi_ext->ref_mvs[ref_frame][0], second_ref, best_yrd,
- &rate, &rate_y, &distortion, &skippable, &total_sse,
- (int)this_rd_thresh, seg_mvs,
-#if CONFIG_EXT_INTER
- compound_seg_newmvs,
-#endif // CONFIG_EXT_INTER
- bsi, switchable_filter_index, mi_row, mi_col);
- if (tmp_rd == INT64_MAX) continue;
- rs = av1_get_switchable_rate(cpi, xd);
- rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
- if (cm->interp_filter == SWITCHABLE) tmp_rd += rs_rd;
-
- newbest = (tmp_rd < tmp_best_rd);
- if (newbest) {
-#if CONFIG_DUAL_FILTER
- tmp_best_filter[0] = mbmi->interp_filter[0];
- tmp_best_filter[1] = mbmi->interp_filter[1];
- tmp_best_filter[2] = mbmi->interp_filter[2];
- tmp_best_filter[3] = mbmi->interp_filter[3];
-#else
- tmp_best_filter = mbmi->interp_filter;
-#endif // CONFIG_DUAL_FILTER
- tmp_best_rd = tmp_rd;
- }
- if ((newbest && cm->interp_filter == SWITCHABLE) ||
- (
-#if CONFIG_DUAL_FILTER
- mbmi->interp_filter[0] == cm->interp_filter
-#else
- mbmi->interp_filter == cm->interp_filter
-#endif // CONFIG_DUAL_FILTER
- && cm->interp_filter != SWITCHABLE)) {
- tmp_best_rdu = tmp_rd;
- tmp_best_rate = rate;
- tmp_best_ratey = rate_y;
- tmp_best_distortion = distortion;
- tmp_best_sse = total_sse;
- tmp_best_skippable = skippable;
- tmp_best_mbmode = *mbmi;
- for (i = 0; i < 4; i++) {
- tmp_best_bmodes[i] = xd->mi[0]->bmi[i];
- }
- pred_exists = 1;
- }
- } // switchable_filter_index loop
- }
- }
-
- if (tmp_best_rdu == INT64_MAX && pred_exists) continue;
-
-#if CONFIG_DUAL_FILTER
- mbmi->interp_filter[0] =
- (cm->interp_filter == SWITCHABLE ? tmp_best_filter[0]
- : cm->interp_filter);
- mbmi->interp_filter[1] =
- (cm->interp_filter == SWITCHABLE ? tmp_best_filter[1]
- : cm->interp_filter);
- mbmi->interp_filter[2] =
- (cm->interp_filter == SWITCHABLE ? tmp_best_filter[2]
- : cm->interp_filter);
- mbmi->interp_filter[3] =
- (cm->interp_filter == SWITCHABLE ? tmp_best_filter[3]
- : cm->interp_filter);
-#else
- mbmi->interp_filter =
- (cm->interp_filter == SWITCHABLE ? tmp_best_filter
- : cm->interp_filter);
-#endif // CONFIG_DUAL_FILTER
-
- if (!pred_exists) {
- // Handles the special case when a filter that is not in the
- // switchable list (bilinear) is indicated at the frame level
- tmp_rd = rd_pick_inter_best_sub8x8_mode(
- cpi, x, &x->mbmi_ext->ref_mvs[ref_frame][0], second_ref, best_yrd,
- &rate, &rate_y, &distortion, &skippable, &total_sse,
- (int)this_rd_thresh, seg_mvs,
-#if CONFIG_EXT_INTER
- compound_seg_newmvs,
-#endif // CONFIG_EXT_INTER
- bsi, 0, mi_row, mi_col);
- if (tmp_rd == INT64_MAX) continue;
- } else {
- total_sse = tmp_best_sse;
- rate = tmp_best_rate;
- rate_y = tmp_best_ratey;
- distortion = tmp_best_distortion;
- skippable = tmp_best_skippable;
- *mbmi = tmp_best_mbmode;
- for (i = 0; i < 4; i++) xd->mi[0]->bmi[i] = tmp_best_bmodes[i];
- }
- // Add in the cost of the transform type
- if (!xd->lossless[mbmi->segment_id]) {
- int rate_tx_type = 0;
-#if CONFIG_EXT_TX
- if (get_ext_tx_types(mbmi->tx_size, bsize, 1, cm->reduced_tx_set_used) >
- 1) {
- const int eset =
- get_ext_tx_set(mbmi->tx_size, bsize, 1, cm->reduced_tx_set_used);
- rate_tx_type =
- cpi->inter_tx_type_costs[eset][mbmi->tx_size][mbmi->tx_type];
- }
-#else
- if (mbmi->tx_size < TX_32X32) {
- rate_tx_type = cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
- }
-#endif // CONFIG_EXT_TX
- rate += rate_tx_type;
- rate_y += rate_tx_type;
- }
-
- rate2 += rate;
- distortion2 += distortion;
-
- if (cm->interp_filter == SWITCHABLE)
- rate2 += av1_get_switchable_rate(cpi, xd);
-
- if (!mode_excluded)
- mode_excluded = comp_pred ? cm->reference_mode == SINGLE_REFERENCE
- : cm->reference_mode == COMPOUND_REFERENCE;
-
- compmode_cost = av1_cost_bit(comp_mode_p, comp_pred);
-
- tmp_best_rdu =
- best_rd - AOMMIN(RDCOST(x->rdmult, x->rddiv, rate2, distortion2),
- RDCOST(x->rdmult, x->rddiv, 0, total_sse));
-
- if (tmp_best_rdu > 0) {
- // If even the 'Y' rd value of split is higher than best so far
- // then dont bother looking at UV
- int is_cost_valid_uv;
- RD_STATS rd_stats_uv;
- av1_build_inter_predictors_sbuv(cm, &x->e_mbd, mi_row, mi_col, NULL,
- BLOCK_8X8);
-#if CONFIG_VAR_TX
- is_cost_valid_uv =
- inter_block_uvrd(cpi, x, &rd_stats_uv, BLOCK_8X8, tmp_best_rdu);
-#else
- is_cost_valid_uv =
- super_block_uvrd(cpi, x, &rd_stats_uv, BLOCK_8X8, tmp_best_rdu);
-#endif // CONFIG_VAR_TX
- rate_uv = rd_stats_uv.rate;
- distortion_uv = rd_stats_uv.dist;
- uv_skippable = rd_stats_uv.skip;
- uv_sse = rd_stats_uv.sse;
-
- if (!is_cost_valid_uv) continue;
- rate2 += rate_uv;
- distortion2 += distortion_uv;
- skippable = skippable && uv_skippable;
- total_sse += uv_sse;
- } else {
- continue;
- }
- }
-
- if (cm->reference_mode == REFERENCE_MODE_SELECT) rate2 += compmode_cost;
-
- // Estimate the reference frame signaling cost and add it
- // to the rolling cost variable.
- if (second_ref_frame > INTRA_FRAME) {
- rate2 += ref_costs_comp[ref_frame];
-#if CONFIG_EXT_REFS
- rate2 += ref_costs_comp[second_ref_frame];
-#endif // CONFIG_EXT_REFS
- } else {
- rate2 += ref_costs_single[ref_frame];
- }
-
- if (!disable_skip) {
- // Skip is never coded at the segment level for sub8x8 blocks and instead
- // always coded in the bitstream at the mode info level.
-
- if (ref_frame != INTRA_FRAME && !xd->lossless[mbmi->segment_id]) {
- if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv, distortion2) <
- RDCOST(x->rdmult, x->rddiv, 0, total_sse)) {
- // Add in the cost of the no skip flag.
- rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0);
- } else {
- // FIXME(rbultje) make this work for splitmv also
- rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 1);
- distortion2 = total_sse;
- assert(total_sse >= 0);
- rate2 -= (rate_y + rate_uv);
- rate_y = 0;
- rate_uv = 0;
- this_skip2 = 1;
- }
- } else {
- // Add in the cost of the no skip flag.
- rate2 += av1_cost_bit(av1_get_skip_prob(cm, xd), 0);
- }
-
- // Calculate the final RD estimate for this mode.
- this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
- }
-
- if (!disable_skip && ref_frame == INTRA_FRAME) {
- for (i = 0; i < REFERENCE_MODES; ++i)
- best_pred_rd[i] = AOMMIN(best_pred_rd[i], this_rd);
- }
-
- // Did this mode help.. i.e. is it the new best mode
- if (this_rd < best_rd || x->skip) {
- if (!mode_excluded) {
- // Note index of best mode so far
- best_ref_index = ref_index;
-
- if (ref_frame == INTRA_FRAME) {
- /* required for left and above block mv */
- mbmi->mv[0].as_int = 0;
- }
-
- rd_cost->rate = rate2;
-#if CONFIG_SUPERTX
- *returnrate_nocoef = rate2 - rate_y - rate_uv;
- if (!disable_skip)
- *returnrate_nocoef -=
- av1_cost_bit(av1_get_skip_prob(cm, xd), this_skip2);
- *returnrate_nocoef -= av1_cost_bit(av1_get_intra_inter_prob(cm, xd),
- mbmi->ref_frame[0] != INTRA_FRAME);
- assert(*returnrate_nocoef > 0);
-#endif // CONFIG_SUPERTX
- rd_cost->dist = distortion2;
- rd_cost->rdcost = this_rd;
- best_rd = this_rd;
- best_yrd =
- best_rd - RDCOST(x->rdmult, x->rddiv, rate_uv, distortion_uv);
- best_mbmode = *mbmi;
- best_skip2 = this_skip2;
-
-#if CONFIG_VAR_TX
- for (i = 0; i < MAX_MB_PLANE; ++i)
- memset(ctx->blk_skip[i], 0, sizeof(uint8_t) * ctx->num_4x4_blk);
-#endif // CONFIG_VAR_TX
-
- for (i = 0; i < 4; i++) best_bmodes[i] = xd->mi[0]->bmi[i];
- }
- }
-
- /* keep record of best compound/single-only prediction */
- if (!disable_skip && ref_frame != INTRA_FRAME) {
- int64_t single_rd, hybrid_rd, single_rate, hybrid_rate;
-
- if (cm->reference_mode == REFERENCE_MODE_SELECT) {
- single_rate = rate2 - compmode_cost;
- hybrid_rate = rate2;
- } else {
- single_rate = rate2;
- hybrid_rate = rate2 + compmode_cost;
- }
-
- single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2);
- hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2);
-
- if (!comp_pred && single_rd < best_pred_rd[SINGLE_REFERENCE])
- best_pred_rd[SINGLE_REFERENCE] = single_rd;
- else if (comp_pred && single_rd < best_pred_rd[COMPOUND_REFERENCE])
- best_pred_rd[COMPOUND_REFERENCE] = single_rd;
-
- if (hybrid_rd < best_pred_rd[REFERENCE_MODE_SELECT])
- best_pred_rd[REFERENCE_MODE_SELECT] = hybrid_rd;
- }
-
- if (x->skip && !comp_pred) break;
- }
-
- if (best_rd >= best_rd_so_far) {
- rd_cost->rate = INT_MAX;
- rd_cost->rdcost = INT64_MAX;
-#if CONFIG_SUPERTX
- *returnrate_nocoef = INT_MAX;
-#endif // CONFIG_SUPERTX
- return;
- }
-
- if (best_rd == INT64_MAX) {
- rd_cost->rate = INT_MAX;
- rd_cost->dist = INT64_MAX;
- rd_cost->rdcost = INT64_MAX;
-#if CONFIG_SUPERTX
- *returnrate_nocoef = INT_MAX;
-#endif // CONFIG_SUPERTX
- return;
- }
-
-#if CONFIG_DUAL_FILTER
- assert((cm->interp_filter == SWITCHABLE) ||
- (cm->interp_filter == best_mbmode.interp_filter[0]) ||
- !is_inter_block(&best_mbmode));
-#else
- assert((cm->interp_filter == SWITCHABLE) ||
- (cm->interp_filter == best_mbmode.interp_filter) ||
- !is_inter_block(&best_mbmode));
-#endif // CONFIG_DUAL_FILTER
-
- av1_update_rd_thresh_fact(cm, tile_data->thresh_freq_fact,
- sf->adaptive_rd_thresh, bsize, best_ref_index);
-
- // macroblock modes
- *mbmi = best_mbmode;
-#if CONFIG_VAR_TX
- mbmi->inter_tx_size[0][0] = mbmi->tx_size;
-#endif // CONFIG_VAR_TX
-
- x->skip |= best_skip2;
- if (!is_inter_block(&best_mbmode)) {
- for (i = 0; i < 4; i++) xd->mi[0]->bmi[i].as_mode = best_bmodes[i].as_mode;
- } else {
- for (i = 0; i < 4; ++i)
- memcpy(&xd->mi[0]->bmi[i], &best_bmodes[i], sizeof(b_mode_info));
-
- mbmi->pred_mv[0].as_int = xd->mi[0]->bmi[3].pred_mv[0].as_int;
- mbmi->pred_mv[1].as_int = xd->mi[0]->bmi[3].pred_mv[1].as_int;
- mbmi->mv[0].as_int = xd->mi[0]->bmi[3].as_mv[0].as_int;
- mbmi->mv[1].as_int = xd->mi[0]->bmi[3].as_mv[1].as_int;
- }
-
-// Note: this section is needed since the mode may have been forced to ZEROMV
-#if CONFIG_GLOBAL_MOTION
- if (mbmi->mode == ZEROMV
-#if CONFIG_EXT_INTER
- || mbmi->mode == ZERO_ZEROMV
-#endif // CONFIG_EXT_INTER
- ) {
- if (is_nontrans_global_motion(xd)) {
-#if CONFIG_DUAL_FILTER
- mbmi->interp_filter[0] = cm->interp_filter == SWITCHABLE
- ? EIGHTTAP_REGULAR
- : cm->interp_filter;
- mbmi->interp_filter[1] = cm->interp_filter == SWITCHABLE
- ? EIGHTTAP_REGULAR
- : cm->interp_filter;
-#else
- mbmi->interp_filter = cm->interp_filter == SWITCHABLE ? EIGHTTAP_REGULAR
- : cm->interp_filter;
-#endif // CONFIG_DUAL_FILTER
- }
- }
-#endif // CONFIG_GLOBAL_MOTION
-
- for (i = 0; i < REFERENCE_MODES; ++i) {
- if (best_pred_rd[i] == INT64_MAX)
- best_pred_diff[i] = INT_MIN;
- else
- best_pred_diff[i] = best_rd - best_pred_rd[i];
- }
-
- store_coding_context(x, ctx, best_ref_index, best_pred_diff, 0);
-}
-
#if CONFIG_MOTION_VAR
// This function has a structure similar to av1_build_obmc_inter_prediction
//
diff --git a/av1/encoder/rdopt.h b/av1/encoder/rdopt.h
index 6563773..e5d778f 100644
--- a/av1/encoder/rdopt.h
+++ b/av1/encoder/rdopt.h
@@ -107,16 +107,6 @@
int av1_active_v_edge(const struct AV1_COMP *cpi, int mi_col, int mi_step);
int av1_active_edge_sb(const struct AV1_COMP *cpi, int mi_row, int mi_col);
-void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi,
- struct TileDataEnc *tile_data,
- struct macroblock *x, int mi_row, int mi_col,
- struct RD_STATS *rd_cost,
-#if CONFIG_SUPERTX
- int *returnrate_nocoef,
-#endif // CONFIG_SUPERTX
- BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx,
- int64_t best_rd_so_far);
-
#if CONFIG_MOTION_VAR && CONFIG_NCOBMC
void av1_check_ncobmc_rd(const struct AV1_COMP *cpi, struct macroblock *x,
int mi_row, int mi_col);
