Misc refactors to support 4:1->2:1->1:1 tx splits Currently 4:1 transforms have max 2 split levels: 4:1 -> 1:1 -> 0.5:0.5. This refactor enables split levels: 4:1 -> 2:1 -> 1:1, by simply changing the tables in common_data.h. The actual switch will be made in a subsequent patch. Change-Id: I33f8d9ca5159ba3e7d02ced449ddf6f804a8f12a
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index c9b54ba..e78ade1 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -1105,23 +1105,34 @@ void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y); -static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize) { - const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize]; - return AOMMIN(tx_size_cat + 1, MAX_TX_DEPTH); +static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize, int is_inter) { + TX_SIZE tx_size = get_max_rect_tx_size(bsize, is_inter); + int depth = 0; + while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) { + depth++; + tx_size = sub_tx_size_map[tx_size]; + } + return depth; } -static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) { - if (tx_size == max_txsize_rect_intra_lookup[bsize]) return 0; - const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize]; - const TX_SIZE coded_tx_size = txsize_sqr_map[tx_size]; - return (int)(tx_size_cat + 1 - (int)coded_tx_size); +static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize, + int is_inter) { + TX_SIZE ctx_size = get_max_rect_tx_size(bsize, is_inter); + int depth = 0; + while (tx_size != ctx_size) { + depth++; + ctx_size = sub_tx_size_map[ctx_size]; + assert(depth <= MAX_TX_DEPTH); + } + return depth; } -static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize) { - if (depth == 0) return max_txsize_rect_intra_lookup[bsize]; - const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize]; - assert(tx_size_cat + 1 - depth >= 0 && tx_size_cat + 1 - depth < TX_SIZES); - return (TX_SIZE)(tx_size_cat + 1 - depth); +static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize, + int is_inter) { + TX_SIZE max_tx_size = get_max_rect_tx_size(bsize, is_inter); + TX_SIZE tx_size = max_tx_size; + for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[tx_size]; + return tx_size; } static INLINE TX_SIZE av1_get_uv_tx_size(const MB_MODE_INFO *mbmi,
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c index c81ddbd..33c71e8 100644 --- a/av1/common/reconintra.c +++ b/av1/common/reconintra.c
@@ -2815,7 +2815,6 @@ // A block should only fail to have a matching transform if it's // large and rectangular (such large transform sizes aren't // available). - assert(block_width >= 32 && block_height >= 32); assert((block_width == wpx && block_height == hpx) || (block_width == (wpx >> 1) && block_height == hpx) || (block_width == wpx && block_height == (hpx >> 1)));
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index ad4d4d1..c6fac1b 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -504,7 +504,7 @@ const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type; const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] : intra_tx_size_cat_lookup[bsize]; - const int max_depths = bsize_to_max_depth(bsize); + const int max_depths = bsize_to_max_depth(bsize, 0); const int ctx = get_tx_size_context(xd); FRAME_CONTEXT *ec_ctx = xd->tile_ctx; (void)cm; @@ -512,7 +512,7 @@ const int depth = aom_read_symbol(r, ec_ctx->tx_size_cdf[tx_size_cat][ctx], max_depths + 1, ACCT_STR); assert(depth >= 0 && depth <= max_depths); - const TX_SIZE tx_size = depth_to_tx_size(depth, bsize); + const TX_SIZE tx_size = depth_to_tx_size(depth, bsize, 0); return tx_size; }
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 607357d..12d69ee 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -263,8 +263,8 @@ const TX_SIZE tx_size = mbmi->tx_size; const int tx_size_ctx = get_tx_size_context(xd); const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize]; - const int depth = tx_size_to_depth(tx_size, bsize); - const int max_depths = bsize_to_max_depth(bsize); + const int depth = tx_size_to_depth(tx_size, bsize, 0); + const int max_depths = bsize_to_max_depth(bsize, 0); assert(depth >= 0 && depth <= max_depths); assert(!is_inter_block(mbmi));
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index bb98ade..1b82279 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -429,11 +429,8 @@ tx_size_from_tx_mode(mbmi->sb_type, tx_mode, is_inter_block(mbmi)); } else { BLOCK_SIZE bsize = mbmi->sb_type; - TX_SIZE max_rect_txsize = get_max_rect_tx_size(bsize, is_inter_block(mbmi)); TX_SIZE min_tx_size = - (TX_SIZE)AOMMAX((int)TX_4X4, - txsize_sqr_map[max_rect_txsize] - MAX_TX_DEPTH + - is_rect_tx(max_rect_txsize)); + depth_to_tx_size(MAX_TX_DEPTH, bsize, is_inter_block(mbmi)); mbmi->tx_size = (TX_SIZE)AOMMAX(mbmi->tx_size, min_tx_size); } } @@ -4538,8 +4535,8 @@ const TX_SIZE tx_size = mbmi->tx_size; const int tx_size_ctx = get_tx_size_context(xd); const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize]; - const int depth = tx_size_to_depth(tx_size, bsize); - const int max_depths = bsize_to_max_depth(bsize); + const int depth = tx_size_to_depth(tx_size, bsize, 0); + const int max_depths = bsize_to_max_depth(bsize, 0); update_cdf(fc->tx_size_cdf[tx_size_cat][tx_size_ctx], depth, max_depths + 1); }
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 5f51fc7..1859e2d 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -2324,7 +2324,7 @@ const int is_inter = is_inter_block(mbmi); const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize] : intra_tx_size_cat_lookup[bsize]; - const int depth = tx_size_to_depth(tx_size, bsize); + const int depth = tx_size_to_depth(tx_size, bsize, is_inter); const int tx_size_ctx = get_tx_size_context(xd); int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth]; return r_tx_size; @@ -2437,7 +2437,6 @@ TX_TYPE tx_type, TX_SIZE tx_size, int prune) { const MACROBLOCKD *const xd = &x->e_mbd; const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; - const TX_SIZE max_tx_size = max_txsize_lookup[bs]; const int is_inter = is_inter_block(mbmi); if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) return 1; @@ -2449,7 +2448,6 @@ if (is_inter && x->use_default_inter_tx_type && tx_type != get_default_tx_type(0, xd, 0, tx_size)) return 1; - if (max_tx_size >= TX_32X32 && tx_size == TX_4X4) return 1; const AV1_COMMON *const cm = &cpi->common; const TxSetType tx_set_type = get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used); @@ -2588,74 +2586,28 @@ MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; int64_t rd = INT64_MAX; int n; - int start_tx, end_tx; + int start_tx; + int depth; int64_t best_rd = INT64_MAX, last_rd = INT64_MAX; - const TX_SIZE max_tx_size = max_txsize_lookup[bs]; - TX_SIZE best_tx_size = max_tx_size; + const int is_inter = is_inter_block(mbmi); + const TX_SIZE max_rect_tx_size = get_max_rect_tx_size(bs, is_inter); + TX_SIZE best_tx_size = max_rect_tx_size; TX_TYPE best_tx_type = DCT_DCT; #if CONFIG_TXK_SEL TX_TYPE best_txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)]; #endif // CONFIG_TXK_SEL const int tx_select = cm->tx_mode == TX_MODE_SELECT; - const int is_inter = is_inter_block(mbmi); av1_invalid_rd_stats(rd_stats); - int evaluate_rect_tx = 0; if (tx_select) { - evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi); - } else { - const TX_SIZE chosen_tx_size = - tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); - evaluate_rect_tx = is_rect_tx(chosen_tx_size); - assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi))); - } - if (evaluate_rect_tx) { - TX_TYPE tx_start = DCT_DCT; - TX_TYPE tx_end = TX_TYPES; -#if CONFIG_TXK_SEL - // The tx_type becomes dummy when lv_map is on. The tx_type search will be - // performed in av1_search_txk_type() - tx_end = DCT_DCT + 1; -#endif - TX_TYPE tx_type; - for (tx_type = tx_start; tx_type < tx_end; ++tx_type) { - if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue; - const TX_SIZE rect_tx_size = get_max_rect_tx_size(bs, is_inter); - RD_STATS this_rd_stats; - const TxSetType tx_set_type = get_ext_tx_set_type( - rect_tx_size, bs, is_inter, cm->reduced_tx_set_used); - if (av1_ext_tx_used[tx_set_type][tx_type]) { - rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, - rect_tx_size); - ref_best_rd = AOMMIN(rd, ref_best_rd); - if (rd < best_rd) { -#if CONFIG_TXK_SEL - memcpy(best_txk_type, mbmi->txk_type, - sizeof(best_txk_type[0]) * MAX_SB_SQUARE / - (TX_SIZE_W_MIN * TX_SIZE_H_MIN)); -#endif - best_tx_type = tx_type; - best_tx_size = rect_tx_size; - best_rd = rd; - *rd_stats = this_rd_stats; - } - } -#if !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 - const int is_inter = is_inter_block(mbmi); - if (mbmi->sb_type < BLOCK_8X8 && is_inter) break; -#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 - } - } - - if (tx_select) { - start_tx = max_tx_size; - end_tx = AOMMAX((int)TX_4X4, start_tx - MAX_TX_DEPTH + evaluate_rect_tx); + start_tx = max_rect_tx_size; + depth = 0; } else { const TX_SIZE chosen_tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); start_tx = chosen_tx_size; - end_tx = chosen_tx_size; + depth = MAX_TX_DEPTH; } int prune = 0; @@ -2665,8 +2617,7 @@ } last_rd = INT64_MAX; - for (n = start_tx; n >= end_tx; --n) { - if (is_rect_tx(n)) break; + for (n = start_tx; depth <= MAX_TX_DEPTH; depth++, n = sub_tx_size_map[n]) { TX_TYPE tx_start = DCT_DCT; TX_TYPE tx_end = TX_TYPES; #if CONFIG_TXK_SEL @@ -2683,8 +2634,8 @@ // Early termination in transform size search. if (cpi->sf.tx_size_search_breakout && (rd == INT64_MAX || - (this_rd_stats.skip == 1 && tx_type != DCT_DCT && n < start_tx) || - (n < (int)max_tx_size && rd > last_rd))) { + (this_rd_stats.skip == 1 && tx_type != DCT_DCT && n != start_tx) || + (n != (int)start_tx && rd > last_rd))) { break; }