Add variance-aware RD cost penalties to avoid texture loss Bias against skip/smooth/compound modes and large MV magnitude for low-to-mid variance blocks in HBD sharpness 3 mode. Bug: 548701298 Change-Id: I8f39cd457acc9f0e3777ad389946a3f41e26dd9b
diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c index db8d9c8..cad0cdc 100644 --- a/av1/encoder/intra_mode_search.c +++ b/av1/encoder/intra_mode_search.c
@@ -998,6 +998,19 @@ intra_mode_info_cost_uv(cpi, x, mbmi, bsize, mode_cost); this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist); +#if CONFIG_AV1_HIGHBITDEPTH + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + const int is_smooth_uv_mode = + (uv_mode == UV_SMOOTH_PRED || uv_mode == UV_SMOOTH_V_PRED || + uv_mode == UV_SMOOTH_H_PRED); + if (is_smooth_uv_mode) { + if (this_rd < INT64_MAX) { + this_rd += (this_rd >> 2); + } + } + } +#endif + if (this_rd < best_rd) { best_mbmi = *mbmi; best_rd = this_rd;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index d6b6e20..053edda 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -736,6 +736,50 @@ return; } +#if CONFIG_AV1_HIGHBITDEPTH + const MACROBLOCKD *xd = &x->e_mbd; + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + if (frame_is_intra_only(&cpi->common)) return; + + int64_t src_var, rec_var; + av1_get_variance_stats(x, &src_var, &rec_var); + + int64_t var_offset = (src_var > rec_var) ? (src_var - rec_var) : 0; + + const MB_MODE_INFO *mbmi = xd->mi[0]; + const int is_smooth_intra_mode = + (mbmi->mode == DC_PRED || mbmi->mode == SMOOTH_PRED || + mbmi->mode == SMOOTH_V_PRED || mbmi->mode == SMOOTH_H_PRED || + mbmi->mode == PAETH_PRED); + + const int is_skip_txfm = mbmi->skip_txfm || rd_cost->skip_txfm; + const int num_pixels = + block_size_wide[mbmi->bsize] * block_size_high[mbmi->bsize]; + const int64_t src_var_per_px = src_var / num_pixels; + + if (var_offset > 0 && + ((is_skip_txfm && src_var_per_px >= 0) || is_smooth_intra_mode || + is_interintra_mode(mbmi) || has_second_ref(mbmi))) { + var_offset *= 4; + } + + if (is_inter_pred && !has_second_ref(mbmi)) { + const int mv_mag = + abs(mbmi->mv[0].as_mv.row) + abs(mbmi->mv[0].as_mv.col); + if (mv_mag > 0 && src_var_per_px < 64) { + var_offset += (int64_t)mv_mag * (64 - src_var_per_px); + } + } + + if (var_offset <= 0) return; + + rd_cost->dist += var_offset; + + rd_cost->rdcost = RDCOST(x->rdmult, rd_cost->rate, rd_cost->dist); + return; + } +#endif + if (cpi->oxcf.algo_cfg.sharpness != 3) return; if (frame_is_kf_gf_arf(cpi)) return; @@ -761,6 +805,48 @@ return; } +#if CONFIG_AV1_HIGHBITDEPTH + const MACROBLOCKD *xd = &x->e_mbd; + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + if (frame_is_intra_only(&cpi->common)) return; + + int64_t src_var, rec_var; + av1_get_variance_stats(x, &src_var, &rec_var); + + int64_t var_offset = (src_var > rec_var) ? (src_var - rec_var) : 0; + + const MB_MODE_INFO *mbmi = xd->mi[0]; + const int is_smooth_intra_mode = + (mbmi->mode == DC_PRED || mbmi->mode == SMOOTH_PRED || + mbmi->mode == SMOOTH_V_PRED || mbmi->mode == SMOOTH_H_PRED || + mbmi->mode == PAETH_PRED); + + const int is_skip_txfm = mbmi->skip_txfm; + const int num_pixels = + block_size_wide[mbmi->bsize] * block_size_high[mbmi->bsize]; + const int64_t src_var_per_px = src_var / num_pixels; + + if (var_offset > 0 && + ((is_skip_txfm && src_var_per_px >= 0) || is_smooth_intra_mode || + is_interintra_mode(mbmi) || has_second_ref(mbmi))) { + var_offset *= 4; + } + + if (is_inter_pred && !has_second_ref(mbmi)) { + const int mv_mag = + abs(mbmi->mv[0].as_mv.row) + abs(mbmi->mv[0].as_mv.col); + if (mv_mag > 0 && src_var_per_px < 64) { + var_offset += (int64_t)mv_mag * (64 - src_var_per_px); + } + } + + if (var_offset <= 0) return; + + *rd_cost += RDCOST(x->rdmult, 0, var_offset); + return; + } +#endif + if (cpi->oxcf.algo_cfg.sharpness != 3) return; if (frame_is_kf_gf_arf(cpi)) return; @@ -1760,6 +1846,8 @@ rd_stats_uv, mbmi); } mbmi->skip_txfm = 0; + rd_stats->skip_txfm = 0; + if (rd_stats_y) rd_stats_y->skip_txfm = 0; increase_motion_mode_rdstats(cpi, mbmi, rd_stats, NULL, NULL); } else { @@ -3901,14 +3989,18 @@ this_sse = rd_stats_y.sse + rd_stats_uv.sse * 15 / 16; } + int64_t no_skip_dist, skip_sse; + av1_get_tx_skip_dist(cpi, x, mbmi->bsize, this_dist, this_sse, + &no_skip_dist, &skip_sse); + if (is_inter_mode(mbmi->mode) && (!cpi->oxcf.algo_cfg.sharpness || !comp_pred) && RDCOST(x->rdmult, mode_costs->skip_txfm_cost[skip_ctx][0] + rd_stats_y.rate + rd_stats_uv.rate, - this_dist) > RDCOST(x->rdmult, - mode_costs->skip_txfm_cost[skip_ctx][1], - this_sse)) { + no_skip_dist) > RDCOST(x->rdmult, + mode_costs->skip_txfm_cost[skip_ctx][1], + skip_sse)) { skip_blk = 1; rd_stats_y.rate = mode_costs->skip_txfm_cost[skip_ctx][1]; rd_stats_uv.rate = 0; @@ -3923,7 +4015,23 @@ &rd_stats_uv); int this_rate = rd_stats.rate + rd_stats_y.rate + rd_stats_uv.rate - winner_rate_y - winner_rate_uv; - int64_t this_rd = RDCOST(x->rdmult, this_rate, this_dist); + int64_t this_rd; +#if CONFIG_AV1_HIGHBITDEPTH + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + RD_STATS tmp_rd_stats; + av1_init_rd_stats(&tmp_rd_stats); + tmp_rd_stats.rate = this_rate; + tmp_rd_stats.dist = this_dist; + tmp_rd_stats.sse = this_sse; + tmp_rd_stats.skip_txfm = skip_blk; + adjust_rdcost(cpi, x, &tmp_rd_stats, is_inter_mode(mbmi->mode)); + this_rd = tmp_rd_stats.rdcost; + this_dist = tmp_rd_stats.dist; + } else +#endif + { + this_rd = RDCOST(x->rdmult, this_rate, this_dist); + } if (best_rd > this_rd) { *best_mbmode = *mbmi; *best_mode_index = winner_mode_index; @@ -4752,6 +4860,7 @@ mbmi->motion_mode = SIMPLE_TRANSLATION; mbmi->interintra_mode = (INTERINTRA_MODE)(II_DC_PRED - 1); set_default_interp_filters(mbmi, cm->features.interp_filter); + mbmi->skip_txfm = 0; } static inline void collect_single_states(MACROBLOCK *x, @@ -5711,7 +5820,13 @@ top_intra_model_rd[i] = INT64_MAX; } - if (cpi->oxcf.algo_cfg.sharpness) { +#if CONFIG_AV1_HIGHBITDEPTH + const int allow_larger_intra = + xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3; +#else + const int allow_larger_intra = 0; +#endif + if (cpi->oxcf.algo_cfg.sharpness && !allow_larger_intra) { int bh = mi_size_high[bsize]; int bw = mi_size_wide[bsize]; if (bh > 4 || bw > 4) return; @@ -6263,6 +6378,7 @@ const int comp_pred = second_ref_frame > INTRA_FRAME; txfm_info->skip_txfm = 0; + mbmi->skip_txfm = 0; sf_args.num_single_modes_processed += is_single_pred; #if CONFIG_COLLECT_COMPONENT_TIMING start_timing(cpi, skip_inter_mode_time); @@ -6348,9 +6464,6 @@ ref_frame_rd[ref_frame] = this_rd; } - adjust_cost(cpi, x, &this_rd, /*is_inter_pred=*/true); - adjust_rdcost(cpi, x, &rd_stats, /*is_inter_pred=*/true); - // Did this mode help, i.e., is it the new best mode if (this_rd < search_state.best_rd) { assert(IMPLIES(comp_pred,