Move av1_search_txk_type() to rdopt.c
Change-Id: I4f9d014324b35e30f25cae5fa570620249640cf6
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 9223b70..9e4ffd4 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -2362,104 +2362,3 @@
assert(0);
}
}
-
-#if CONFIG_TXK_SEL
-int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
- int block, int blk_row, int blk_col,
- BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
- const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l,
- int use_fast_coef_costing, RD_STATS *rd_stats) {
- const AV1_COMMON *cm = &cpi->common;
- MACROBLOCKD *xd = &x->e_mbd;
- MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
- TX_TYPE txk_start = DCT_DCT;
- TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
- TX_TYPE best_tx_type = txk_start;
- int64_t best_rd = INT64_MAX;
- uint8_t best_txb_ctx = 0;
- uint16_t best_eob = 0;
- RD_STATS best_rd_stats;
- TX_TYPE tx_type;
- int rate_cost = 0;
-
- av1_invalid_rd_stats(&best_rd_stats);
-
- for (tx_type = txk_start; tx_type <= txk_end; ++tx_type) {
- if (plane == 0)
- mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type;
- TX_TYPE ref_tx_type =
- av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size);
- if (tx_type != ref_tx_type) {
- // use av1_get_tx_type() to check if the tx_type is valid for the current
- // mode if it's not, we skip it here.
- continue;
- }
-
- RD_STATS this_rd_stats;
- av1_invalid_rd_stats(&this_rd_stats);
- if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
- av1_xform_quant(
- cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
- USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
- } else {
- av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
- tx_size, AV1_XFORM_QUANT_FP);
- av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
- tx_size, a, l, 1, &rate_cost);
- }
- av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size,
- &this_rd_stats.dist, &this_rd_stats.sse,
- OUTPUT_HAS_PREDICTED_PIXELS);
-
- const int eob = x->plane[plane].eobs[block];
- const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
- if (eob)
- rate_cost +=
- av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
- else
- rate_cost =
- av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size,
- scan_order, a, l, use_fast_coef_costing);
- this_rd_stats.rate = rate_cost;
-
- int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist);
-
- if (rd < best_rd) {
- best_rd = rd;
- best_rd_stats = this_rd_stats;
- best_tx_type = tx_type;
- best_txb_ctx = x->plane[plane].txb_entropy_ctx[block];
- best_eob = x->plane[plane].eobs[block];
- }
- }
-
- av1_merge_rd_stats(rd_stats, &best_rd_stats);
-
- if (best_eob == 0) best_tx_type = DCT_DCT;
-
- if (plane == 0)
- mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type;
- x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx;
- x->plane[plane].eobs[block] = best_eob;
-
- if (!is_inter_block(mbmi) && best_eob) {
- // intra mode needs decoded result such that the next transform block
- // can use it for prediction.
- if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
- av1_xform_quant(
- cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
- USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
- } else {
- av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
- tx_size, AV1_XFORM_QUANT_FP);
- av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
- tx_size, a, l, 1, &rate_cost);
- }
-
- av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col,
- x->plane[plane].eobs[block],
- cm->reduced_tx_set_used);
- }
- return best_rd;
-}
-#endif // CONFIG_TXK_SEL
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h
index b867464..714b16c 100644
--- a/av1/encoder/encodetxb.h
+++ b/av1/encoder/encodetxb.h
@@ -94,14 +94,6 @@
void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x,
int mi_row, int mi_col);
-#if CONFIG_TXK_SEL
-int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
- int block, int blk_row, int blk_col,
- BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
- const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l,
- int use_fast_coef_costing, RD_STATS *rd_stats);
-#endif
-
int av1_optimize_txb(const AV1_COMP *cpi, MACROBLOCK *x, int plane, int blk_row,
int blk_col, int block, TX_SIZE tx_size, TXB_CTX *txb_ctx,
int fast_mode, int *rate_cost);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 2fb4340..97f586e 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1817,6 +1817,106 @@
}
}
+#if CONFIG_TXK_SEL
+static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
+ int block, int blk_row, int blk_col,
+ BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
+ const ENTROPY_CONTEXT *a,
+ const ENTROPY_CONTEXT *l,
+ int use_fast_coef_costing, RD_STATS *rd_stats) {
+ const AV1_COMMON *cm = &cpi->common;
+ MACROBLOCKD *xd = &x->e_mbd;
+ MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
+ int rate_cost = 0;
+ const int is_inter = is_inter_block(mbmi);
+ TX_TYPE txk_start = DCT_DCT;
+ TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
+ TX_TYPE best_tx_type = txk_start;
+ int64_t best_rd = INT64_MAX;
+ uint8_t best_txb_ctx = 0;
+ uint16_t best_eob = 0;
+ RD_STATS best_rd_stats;
+ av1_invalid_rd_stats(&best_rd_stats);
+ for (TX_TYPE tx_type = txk_start; tx_type <= txk_end; ++tx_type) {
+ if (plane == 0)
+ mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type;
+ TX_TYPE ref_tx_type =
+ av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size);
+ if (tx_type != ref_tx_type) {
+ // use av1_get_tx_type() to check if the tx_type is valid for the current
+ // mode if it's not, we skip it here.
+ continue;
+ }
+
+ RD_STATS this_rd_stats;
+ av1_invalid_rd_stats(&this_rd_stats);
+ if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+ av1_xform_quant(
+ cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+ USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
+ } else {
+ av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
+ tx_size, AV1_XFORM_QUANT_FP);
+ av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
+ tx_size, a, l, 1, &rate_cost);
+ }
+ av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size,
+ &this_rd_stats.dist, &this_rd_stats.sse,
+ OUTPUT_HAS_PREDICTED_PIXELS);
+
+ const int eob = x->plane[plane].eobs[block];
+ const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
+ if (eob)
+ rate_cost +=
+ av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
+ else
+ rate_cost =
+ av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size,
+ scan_order, a, l, use_fast_coef_costing);
+ this_rd_stats.rate = rate_cost;
+
+ int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist);
+
+ if (rd < best_rd) {
+ best_rd = rd;
+ best_rd_stats = this_rd_stats;
+ best_tx_type = tx_type;
+ best_txb_ctx = x->plane[plane].txb_entropy_ctx[block];
+ best_eob = x->plane[plane].eobs[block];
+ }
+ }
+
+ av1_merge_rd_stats(rd_stats, &best_rd_stats);
+
+ if (best_eob == 0) best_tx_type = DCT_DCT;
+
+ if (plane == 0)
+ mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type;
+ x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx;
+ x->plane[plane].eobs[block] = best_eob;
+
+ if (!is_inter && best_eob) {
+ // intra mode needs decoded result such that the next transform block
+ // can use it for prediction.
+ if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+ av1_xform_quant(
+ cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+ USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
+ } else {
+ av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
+ tx_size, AV1_XFORM_QUANT_FP);
+ av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
+ tx_size, a, l, 1, &rate_cost);
+ }
+
+ av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col,
+ x->plane[plane].eobs[block],
+ cm->reduced_tx_set_used);
+ }
+ return best_rd;
+}
+#endif // CONFIG_TXK_SEL
+
static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
struct rdcost_block_args *args = arg;
@@ -1935,9 +2035,8 @@
this_rd_stats.rate = rate_cost;
#else // !CONFIG_TXK_SEL
- av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
- tx_size, a, l, args->use_fast_coef_costing,
- &this_rd_stats);
+ search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+ a, l, args->use_fast_coef_costing, &this_rd_stats);
#endif // !CONFIG_TXK_SEL
#if CONFIG_CFL
@@ -2438,7 +2537,7 @@
TX_TYPE tx_end = TX_TYPES;
#if CONFIG_TXK_SEL
// The tx_type becomes dummy when lv_map is on. The tx_type search will be
- // performed in av1_search_txk_type()
+ // performed in search_txk_type()
tx_end = DCT_DCT + 1;
#endif
TX_TYPE tx_type;
@@ -3521,8 +3620,8 @@
const struct macroblock_plane *const p = &x->plane[plane];
struct macroblockd_plane *const pd = &xd->plane[plane];
#if CONFIG_TXK_SEL
- av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
- tx_size, a, l, 0, rd_stats);
+ search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+ a, l, 0, rd_stats);
return;
#endif
// This function is used only for inter
