Move av1_search_txk_type() to rdopt.c Change-Id: I4f9d014324b35e30f25cae5fa570620249640cf6
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index 9223b70..9e4ffd4 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -2362,104 +2362,3 @@ assert(0); } } - -#if CONFIG_TXK_SEL -int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, - int block, int blk_row, int blk_col, - BLOCK_SIZE plane_bsize, TX_SIZE tx_size, - const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, - int use_fast_coef_costing, RD_STATS *rd_stats) { - const AV1_COMMON *cm = &cpi->common; - MACROBLOCKD *xd = &x->e_mbd; - MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; - TX_TYPE txk_start = DCT_DCT; - TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1; - TX_TYPE best_tx_type = txk_start; - int64_t best_rd = INT64_MAX; - uint8_t best_txb_ctx = 0; - uint16_t best_eob = 0; - RD_STATS best_rd_stats; - TX_TYPE tx_type; - int rate_cost = 0; - - av1_invalid_rd_stats(&best_rd_stats); - - for (tx_type = txk_start; tx_type <= txk_end; ++tx_type) { - if (plane == 0) - mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type; - TX_TYPE ref_tx_type = - av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size); - if (tx_type != ref_tx_type) { - // use av1_get_tx_type() to check if the tx_type is valid for the current - // mode if it's not, we skip it here. - continue; - } - - RD_STATS this_rd_stats; - av1_invalid_rd_stats(&this_rd_stats); - if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) { - av1_xform_quant( - cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, - USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP); - } else { - av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, AV1_XFORM_QUANT_FP); - av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize, - tx_size, a, l, 1, &rate_cost); - } - av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size, - &this_rd_stats.dist, &this_rd_stats.sse, - OUTPUT_HAS_PREDICTED_PIXELS); - - const int eob = x->plane[plane].eobs[block]; - const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi); - if (eob) - rate_cost += - av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type); - else - rate_cost = - av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size, - scan_order, a, l, use_fast_coef_costing); - this_rd_stats.rate = rate_cost; - - int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist); - - if (rd < best_rd) { - best_rd = rd; - best_rd_stats = this_rd_stats; - best_tx_type = tx_type; - best_txb_ctx = x->plane[plane].txb_entropy_ctx[block]; - best_eob = x->plane[plane].eobs[block]; - } - } - - av1_merge_rd_stats(rd_stats, &best_rd_stats); - - if (best_eob == 0) best_tx_type = DCT_DCT; - - if (plane == 0) - mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type; - x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx; - x->plane[plane].eobs[block] = best_eob; - - if (!is_inter_block(mbmi) && best_eob) { - // intra mode needs decoded result such that the next transform block - // can use it for prediction. - if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) { - av1_xform_quant( - cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, - USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP); - } else { - av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, AV1_XFORM_QUANT_FP); - av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize, - tx_size, a, l, 1, &rate_cost); - } - - av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col, - x->plane[plane].eobs[block], - cm->reduced_tx_set_used); - } - return best_rd; -} -#endif // CONFIG_TXK_SEL
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h index b867464..714b16c 100644 --- a/av1/encoder/encodetxb.h +++ b/av1/encoder/encodetxb.h
@@ -94,14 +94,6 @@ void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x, int mi_row, int mi_col); -#if CONFIG_TXK_SEL -int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, - int block, int blk_row, int blk_col, - BLOCK_SIZE plane_bsize, TX_SIZE tx_size, - const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l, - int use_fast_coef_costing, RD_STATS *rd_stats); -#endif - int av1_optimize_txb(const AV1_COMP *cpi, MACROBLOCK *x, int plane, int blk_row, int blk_col, int block, TX_SIZE tx_size, TXB_CTX *txb_ctx, int fast_mode, int *rate_cost);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 2fb4340..97f586e 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -1817,6 +1817,106 @@ } } +#if CONFIG_TXK_SEL +static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, + int block, int blk_row, int blk_col, + BLOCK_SIZE plane_bsize, TX_SIZE tx_size, + const ENTROPY_CONTEXT *a, + const ENTROPY_CONTEXT *l, + int use_fast_coef_costing, RD_STATS *rd_stats) { + const AV1_COMMON *cm = &cpi->common; + MACROBLOCKD *xd = &x->e_mbd; + MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; + int rate_cost = 0; + const int is_inter = is_inter_block(mbmi); + TX_TYPE txk_start = DCT_DCT; + TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1; + TX_TYPE best_tx_type = txk_start; + int64_t best_rd = INT64_MAX; + uint8_t best_txb_ctx = 0; + uint16_t best_eob = 0; + RD_STATS best_rd_stats; + av1_invalid_rd_stats(&best_rd_stats); + for (TX_TYPE tx_type = txk_start; tx_type <= txk_end; ++tx_type) { + if (plane == 0) + mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type; + TX_TYPE ref_tx_type = + av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size); + if (tx_type != ref_tx_type) { + // use av1_get_tx_type() to check if the tx_type is valid for the current + // mode if it's not, we skip it here. + continue; + } + + RD_STATS this_rd_stats; + av1_invalid_rd_stats(&this_rd_stats); + if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) { + av1_xform_quant( + cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP); + } else { + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, AV1_XFORM_QUANT_FP); + av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize, + tx_size, a, l, 1, &rate_cost); + } + av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size, + &this_rd_stats.dist, &this_rd_stats.sse, + OUTPUT_HAS_PREDICTED_PIXELS); + + const int eob = x->plane[plane].eobs[block]; + const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi); + if (eob) + rate_cost += + av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type); + else + rate_cost = + av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size, + scan_order, a, l, use_fast_coef_costing); + this_rd_stats.rate = rate_cost; + + int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist); + + if (rd < best_rd) { + best_rd = rd; + best_rd_stats = this_rd_stats; + best_tx_type = tx_type; + best_txb_ctx = x->plane[plane].txb_entropy_ctx[block]; + best_eob = x->plane[plane].eobs[block]; + } + } + + av1_merge_rd_stats(rd_stats, &best_rd_stats); + + if (best_eob == 0) best_tx_type = DCT_DCT; + + if (plane == 0) + mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type; + x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx; + x->plane[plane].eobs[block] = best_eob; + + if (!is_inter && best_eob) { + // intra mode needs decoded result such that the next transform block + // can use it for prediction. + if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) { + av1_xform_quant( + cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP); + } else { + av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, + tx_size, AV1_XFORM_QUANT_FP); + av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize, + tx_size, a, l, 1, &rate_cost); + } + + av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col, + x->plane[plane].eobs[block], + cm->reduced_tx_set_used); + } + return best_rd; +} +#endif // CONFIG_TXK_SEL + static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) { struct rdcost_block_args *args = arg; @@ -1935,9 +2035,8 @@ this_rd_stats.rate = rate_cost; #else // !CONFIG_TXK_SEL - av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, a, l, args->use_fast_coef_costing, - &this_rd_stats); + search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + a, l, args->use_fast_coef_costing, &this_rd_stats); #endif // !CONFIG_TXK_SEL #if CONFIG_CFL @@ -2438,7 +2537,7 @@ TX_TYPE tx_end = TX_TYPES; #if CONFIG_TXK_SEL // The tx_type becomes dummy when lv_map is on. The tx_type search will be - // performed in av1_search_txk_type() + // performed in search_txk_type() tx_end = DCT_DCT + 1; #endif TX_TYPE tx_type; @@ -3521,8 +3620,8 @@ const struct macroblock_plane *const p = &x->plane[plane]; struct macroblockd_plane *const pd = &xd->plane[plane]; #if CONFIG_TXK_SEL - av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, - tx_size, a, l, 0, rd_stats); + search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, + a, l, 0, rd_stats); return; #endif // This function is used only for inter