Modify choose_tx_size_type_from_rd for lv_map exp
The tx_type becomes dummy when lv_map is on. The tx_type search
will be performed in av1_search_txk_type() and the results will be
stored in txk_type[]
In choose_tx_size_type_from_rd, we use best_txk_type[] to record
the best search result within different tx_size.
Change-Id: I67c489d261aecaa0092e0cc562670588c6c55941
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index f026d9b..c86a4d0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2173,6 +2173,13 @@
cpi->sf.use_fast_coef_costing);
}
+#if CONFIG_LV_MAP || CONFIG_VAR_TX
+static INLINE int bsize_to_num_blk(BLOCK_SIZE bsize) {
+ int num_blk = 1 << (num_pels_log2_lookup[bsize] - 2 * tx_size_wide_log2[0]);
+ return num_blk;
+}
+#endif // CONFIG_LV_MAP || CONFIG_VAR_TX
+
static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
MACROBLOCK *x, RD_STATS *rd_stats,
int64_t ref_best_rd, BLOCK_SIZE bs) {
@@ -2186,6 +2193,10 @@
const TX_SIZE max_tx_size = max_txsize_lookup[bs];
TX_SIZE best_tx_size = max_tx_size;
TX_TYPE best_tx_type = DCT_DCT;
+#if CONFIG_LV_MAP
+ TX_TYPE best_txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
+ const int num_blk = bsize_to_num_blk(bs);
+#endif // !CONFIG_LV_MAP
const int tx_select = cm->tx_mode == TX_MODE_SELECT;
const int is_inter = is_inter_block(mbmi);
#if CONFIG_PVQ
@@ -2206,8 +2217,15 @@
assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi)));
}
if (evaluate_rect_tx) {
+ TX_TYPE tx_start = DCT_DCT;
+ TX_TYPE tx_end = TX_TYPES;
+#if CONFIG_LV_MAP
+ // The tx_type becomes dummy when lv_map is on. The tx_type search will be
+ // performed in av1_search_txk_type()
+ tx_end = DCT_DCT + 1;
+#endif
TX_TYPE tx_type;
- for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
+ for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
#if CONFIG_REF_MV
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
#endif // CONFIG_REF_MV
@@ -2220,6 +2238,10 @@
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type,
rect_tx_size);
if (rd < best_rd) {
+#if CONFIG_LV_MAP
+ memcpy(best_txk_type, mbmi->txk_type,
+ sizeof(best_txk_type[0]) * num_blk);
+#endif
best_tx_type = tx_type;
best_tx_size = rect_tx_size;
best_rd = rd;
@@ -2246,8 +2268,15 @@
last_rd = INT64_MAX;
for (n = start_tx; n >= end_tx; --n) {
+ TX_TYPE tx_start = DCT_DCT;
+ TX_TYPE tx_end = TX_TYPES;
+#if CONFIG_LV_MAP
+ // The tx_type becomes dummy when lv_map is on. The tx_type search will be
+ // performed in av1_search_txk_type()
+ tx_end = DCT_DCT + 1;
+#endif
TX_TYPE tx_type;
- for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
+ for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
RD_STATS this_rd_stats;
if (skip_txfm_search(cpi, x, bs, tx_type, n)) continue;
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, n);
@@ -2263,6 +2292,10 @@
last_rd = rd;
if (rd < best_rd) {
+#if CONFIG_LV_MAP
+ memcpy(best_txk_type, mbmi->txk_type,
+ sizeof(best_txk_type[0]) * num_blk);
+#endif
best_tx_type = tx_type;
best_tx_size = n;
best_rd = rd;
@@ -2276,6 +2309,9 @@
}
mbmi->tx_size = best_tx_size;
mbmi->tx_type = best_tx_type;
+#if CONFIG_LV_MAP
+ memcpy(mbmi->txk_type, best_txk_type, sizeof(best_txk_type[0]) * num_blk);
+#endif
#if CONFIG_VAR_TX
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
@@ -4227,7 +4263,7 @@
TX_SIZE best_tx = max_txsize_lookup[bsize];
TX_SIZE best_min_tx_size = TX_SIZES_ALL;
uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
- const int n4 = 1 << (num_pels_log2_lookup[bsize] - 2 * tx_size_wide_log2[0]);
+ const int n4 = bsize_to_num_blk(bsize);
int idx, idy;
int prune = 0;
const int count32 =
