Adjust transform search and skip decisions for HBD sharpness Disable pre-skip prediction when reconstructed variance drops below source variance, and bias against skip txfm when skipping the transform causes texutre loss. Bug: 548701298 Change-Id: Idd0298cf4f912880ec68c79f02c840677ae98dde
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index b0ea574..d6b6e20 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -644,8 +644,8 @@ } #endif // CONFIG_AV1_HIGHBITDEPTH -static void get_variance_stats(const MACROBLOCK *x, int64_t *src_var, - int64_t *rec_var) { +void av1_get_variance_stats(const MACROBLOCK *x, int64_t *src_var, + int64_t *rec_var) { #if CONFIG_AV1_HIGHBITDEPTH if (is_cur_buf_hbd(&x->e_mbd)) { get_variance_stats_hbd(x, src_var, rec_var); @@ -665,6 +665,54 @@ *src_var = aom_calc_variance_stat(p->src.buf, p->src.stride, bw, bh); } +int av1_is_skip_txfm_penalized(const AV1_COMP *cpi, const MACROBLOCK *x, + BLOCK_SIZE bsize) { +#if CONFIG_AV1_HIGHBITDEPTH + const MACROBLOCKD *xd = &x->e_mbd; + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3 && + !frame_is_kf_gf_arf(cpi)) { + int64_t src_var, rec_var; + av1_get_variance_stats(x, &src_var, &rec_var); + if (src_var > rec_var) { + const int num_pixels = block_size_wide[bsize] * block_size_high[bsize]; + const int64_t src_var_per_px = src_var / num_pixels; + if (src_var_per_px >= 0) return 1; + } + } +#else + (void)cpi; + (void)x; + (void)bsize; +#endif + return 0; +} + +void av1_get_tx_skip_dist(const AV1_COMP *cpi, const MACROBLOCK *x, + BLOCK_SIZE bsize, int64_t dist, int64_t sse, + int64_t *no_skip_dist, int64_t *skip_dist) { + *no_skip_dist = dist; + *skip_dist = sse; +#if CONFIG_AV1_HIGHBITDEPTH + const MACROBLOCKD *xd = &x->e_mbd; + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3 && + !frame_is_kf_gf_arf(cpi)) { + int64_t src_var, rec_var; + av1_get_variance_stats(x, &src_var, &rec_var); + if (src_var > rec_var) { + int64_t var_offset = src_var - rec_var; + const int num_pixels = block_size_wide[bsize] * block_size_high[bsize]; + const int64_t src_var_per_px = src_var / num_pixels; + *no_skip_dist += var_offset; + *skip_dist += (src_var_per_px >= 0) ? (var_offset * 4) : var_offset; + } + } +#else + (void)cpi; + (void)x; + (void)bsize; +#endif +} + static void adjust_rdcost(const AV1_COMP *cpi, const MACROBLOCK *x, RD_STATS *rd_cost, bool is_inter_pred) { if ((cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ || @@ -693,7 +741,7 @@ if (frame_is_kf_gf_arf(cpi)) return; int64_t src_var, rec_var; - get_variance_stats(x, &src_var, &rec_var); + av1_get_variance_stats(x, &src_var, &rec_var); if (src_var <= rec_var) return; @@ -718,7 +766,7 @@ if (frame_is_kf_gf_arf(cpi)) return; int64_t src_var, rec_var; - get_variance_stats(x, &src_var, &rec_var); + av1_get_variance_stats(x, &src_var, &rec_var); if (src_var <= rec_var) return;
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h index 04d46d0..2082163 100644 --- a/av1/encoder/rdopt_utils.h +++ b/av1/encoder/rdopt_utils.h
@@ -901,6 +901,16 @@ return txfm_rd_gate_level[TX_SEARCH_DEFAULT]; } +void av1_get_variance_stats(const MACROBLOCK *x, int64_t *src_var, + int64_t *rec_var); + +void av1_get_tx_skip_dist(const struct AV1_COMP *cpi, const MACROBLOCK *x, + BLOCK_SIZE bsize, int64_t dist, int64_t sse, + int64_t *no_skip_dist, int64_t *skip_dist); + +int av1_is_skip_txfm_penalized(const struct AV1_COMP *cpi, const MACROBLOCK *x, + BLOCK_SIZE bsize); + #ifdef __cplusplus } // extern "C" #endif
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c index 70e862c..6281191 100644 --- a/av1/encoder/tx_search.c +++ b/av1/encoder/tx_search.c
@@ -190,12 +190,20 @@ // Uses simple features on top of DCT coefficients to quickly predict // whether optimal RD decision is to skip encoding the residual. // The sse value is stored in dist. -static int predict_skip_txfm(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist, +static int predict_skip_txfm(const AV1_COMP *cpi, MACROBLOCK *x, + BLOCK_SIZE bsize, int64_t *dist, int reduced_tx_set) { + const MACROBLOCKD *xd = &x->e_mbd; +#if CONFIG_AV1_HIGHBITDEPTH + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + if (av1_is_skip_txfm_penalized(cpi, x, bsize)) return 0; + } +#else + (void)cpi; +#endif const TxfmSearchParams *txfm_params = &x->txfm_search_params; const int bw = block_size_wide[bsize]; const int bh = block_size_high[bsize]; - const MACROBLOCKD *xd = &x->e_mbd; const int16_t dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd); *dist = av1_pixel_diff_dist(x, 0, 0, 0, bsize, bsize, NULL); @@ -2953,13 +2961,33 @@ } // Check if forcing the block to skip transform leads to smaller RD cost. if (is_inter && !rd_stats->skip_txfm && !xd->lossless[mbmi->segment_id]) { - int64_t temp_skip_txfm_rd = - RDCOST(x->rdmult, skip_txfm_rate, rd_stats->sse); - if (temp_skip_txfm_rd <= rd) { - rd = temp_skip_txfm_rd; - rd_stats->rate = 0; - rd_stats->dist = rd_stats->sse; - rd_stats->skip_txfm = 1; +#if CONFIG_AV1_HIGHBITDEPTH + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + int64_t no_skip_dist, skip_dist; + av1_get_tx_skip_dist(cpi, x, bs, rd_stats->dist, rd_stats->sse, + &no_skip_dist, &skip_dist); + const int64_t temp_skip_txfm_rd = + RDCOST(x->rdmult, skip_txfm_rate, skip_dist); + const int64_t temp_no_skip_txfm_rd = + RDCOST(x->rdmult, rd_stats->rate + no_skip_txfm_rate + tx_size_rate, + no_skip_dist); + if (temp_skip_txfm_rd <= temp_no_skip_txfm_rd) { + rd = temp_skip_txfm_rd; + rd_stats->rate = 0; + rd_stats->dist = rd_stats->sse; + rd_stats->skip_txfm = 1; + } + } else +#endif + { + const int64_t temp_skip_txfm_rd = + RDCOST(x->rdmult, skip_txfm_rate, rd_stats->sse); + if (temp_skip_txfm_rd <= rd) { + rd = temp_skip_txfm_rd; + rd_stats->rate = 0; + rd_stats->dist = rd_stats->sse; + rd_stats->skip_txfm = 1; + } } } @@ -3600,7 +3628,7 @@ // context and terminate early. int64_t dist; if (txfm_params->skip_txfm_level && - predict_skip_txfm(x, bsize, &dist, + predict_skip_txfm(cpi, x, bsize, &dist, cpi->common.features.reduced_tx_set_used)) { set_skip_txfm(x, rd_stats, bsize, dist); // Save the RD search results into mb_rd_record. @@ -3673,7 +3701,7 @@ int64_t dist; if (tx_params->skip_txfm_level && is_inter && !xd->lossless[mbmi->segment_id] && - predict_skip_txfm(x, bs, &dist, + predict_skip_txfm(cpi, x, bs, &dist, cpi->common.features.reduced_tx_set_used)) { // Populate rdstats as per skip decision set_skip_txfm(x, rd_stats, bs, dist); @@ -3874,14 +3902,23 @@ int choose_skip_txfm = rd_stats->skip_txfm; if (!choose_skip_txfm && !xd->lossless[mbmi->segment_id]) { + int64_t no_skip_dist = rd_stats->dist; + int64_t skip_dist = rd_stats->sse; +#if CONFIG_AV1_HIGHBITDEPTH + if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) { + av1_get_tx_skip_dist(cpi, x, bsize, rd_stats->dist, rd_stats->sse, + &no_skip_dist, &skip_dist); + } +#endif const int64_t rdcost_no_skip_txfm = RDCOST( x->rdmult, rd_stats_y->rate + rd_stats_uv->rate + skip_txfm_cost[0], - rd_stats->dist); + no_skip_dist); const int64_t rdcost_skip_txfm = - RDCOST(x->rdmult, skip_txfm_cost[1], rd_stats->sse); + RDCOST(x->rdmult, skip_txfm_cost[1], skip_dist); if (rdcost_no_skip_txfm >= rdcost_skip_txfm) choose_skip_txfm = 1; } if (choose_skip_txfm) { + const int naturally_skip = rd_stats->skip_txfm; rd_stats_y->rate = 0; rd_stats_uv->rate = 0; rd_stats->rate = mode_rate + skip_txfm_cost[1]; @@ -3889,13 +3926,15 @@ rd_stats_y->dist = rd_stats_y->sse; rd_stats_uv->dist = rd_stats_uv->sse; mbmi->skip_txfm = 1; - if (rd_stats->skip_txfm) { + rd_stats->skip_txfm = 1; + if (naturally_skip) { const int64_t tmprd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); if (tmprd > ref_best_rd) return 0; } } else { rd_stats->rate += skip_txfm_cost[0]; mbmi->skip_txfm = 0; + rd_stats->skip_txfm = 0; } return 1;