rect_tx_ext: work with var_tx Change-Id: Ie2c34490dc50cb242bcd701308e6b55243883b15
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 4ba0def..8ea6462 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -1085,8 +1085,10 @@ return is_rect_tx_allowed_bsize(mbmi->sb_type) && !xd->lossless[mbmi->segment_id]; } +#endif // CONFIG_RECT_TX +#endif // CONFIG_EXT_TX -#if CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) static INLINE int is_quarter_tx_allowed_bsize(BLOCK_SIZE bsize) { static const char LUT_QTTX[BLOCK_SIZES_ALL] = { #if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8 @@ -1127,9 +1129,7 @@ return is_quarter_tx_allowed_bsize(mbmi->sb_type) && is_inter && !xd->lossless[mbmi->segment_id]; } -#endif // CONFIG_RECT_TX_EXT -#endif // CONFIG_RECT_TX -#endif // CONFIG_EXT_TX +#endif static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode, int is_inter) {
diff --git a/av1/common/common_data.h b/av1/common/common_data.h index 3aa437d..f49c733 100644 --- a/av1/common/common_data.h +++ b/av1/common/common_data.h
@@ -655,13 +655,20 @@ TX_32X32, TX_32X32, TX_32X32, #endif // CONFIG_EXT_PARTITION #endif // CONFIG_TX64X64 +#if CONFIG_RECT_TX_EXT + // 4x16, 16x4, 8x32 + TX_4X16, TX_16X4, TX_8X32, + // 32x8 + TX_32X8 +#else // 4x16, 16x4, 8x32 TX_4X8, TX_8X4, TX_8X16, // 32x8 TX_16X8 +#endif }; -#if CONFIG_EXT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT static const TX_SIZE quarter_txsize_lookup[BLOCK_SIZES_ALL] = { #if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8 // 2X2, 2X4, 4X2, @@ -686,7 +693,7 @@ // 32x8 TX_32X8 }; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX_EXT +#endif #else #define max_txsize_rect_lookup max_txsize_lookup #endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
diff --git a/av1/common/entropy.h b/av1/common/entropy.h index 51803db..190b792 100644 --- a/av1/common/entropy.h +++ b/av1/common/entropy.h
@@ -315,7 +315,7 @@ *(const uint64_t *)(l + 16) | *(const uint64_t *)(l + 24)); break; #endif // CONFIG_TX64X64 -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) case TX_4X16: above_ec = !!*(const uint16_t *)a; left_ec = !!*(const uint64_t *)l; @@ -332,7 +332,7 @@ above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8)); left_ec = !!*(const uint32_t *)l; break; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif default: assert(0 && "Invalid transform size."); break; } return combine_entropy_contexts(above_ec, left_ec); @@ -385,7 +385,7 @@ left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8)); break; #endif // CONFIG_TX64X64 -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) case TX_4X16: above_ec = a[0] != 0; left_ec = !!*(const uint32_t *)l; @@ -402,7 +402,7 @@ above_ec = !!*(const uint64_t *)a; left_ec = !!*(const uint16_t *)l; break; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif default: assert(0 && "Invalid transform size."); break; } return combine_entropy_contexts(above_ec, left_ec);
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c index c4d75c5..9faa03e 100644 --- a/av1/common/entropymode.c +++ b/av1/common/entropymode.c
@@ -1737,9 +1737,9 @@ #endif // CONFIG_TX64X64 }; -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) static const aom_prob default_quarter_tx_size_prob = 192; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_LOOP_RESTORATION const aom_tree_index @@ -4952,9 +4952,9 @@ av1_copy(fc->comp_inter_mode_prob, default_comp_inter_mode_p); #endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF av1_copy(fc->tx_size_probs, default_tx_size_prob); -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) fc->quarter_tx_size_prob = default_quarter_tx_size_prob; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_VAR_TX av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs); #if CONFIG_NEW_MULTISYMBOL @@ -5231,10 +5231,10 @@ aom_tree_merge_probs(av1_tx_size_tree[i], pre_fc->tx_size_probs[i][j], counts->tx_size[i][j], fc->tx_size_probs[i][j]); } -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) fc->quarter_tx_size_prob = av1_mode_mv_merge_probs( pre_fc->quarter_tx_size_prob, counts->quarter_tx_size); -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif } #if CONFIG_VAR_TX
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h index 83beb6a..3253247 100644 --- a/av1/common/entropymode.h +++ b/av1/common/entropymode.h
@@ -277,9 +277,9 @@ aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS]; #endif // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH]; -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) aom_prob quarter_tx_size_prob; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_VAR_TX aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS]; #if CONFIG_NEW_MULTISYMBOL @@ -467,9 +467,9 @@ // belong into this structure. unsigned int tx_size_totals[TX_SIZES]; unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1]; -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) unsigned int quarter_tx_size[2]; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_VAR_TX unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2]; #endif
diff --git a/av1/common/idct.c b/av1/common/idct.c index 09f01d3..7915b70 100644 --- a/av1/common/idct.c +++ b/av1/common/idct.c
@@ -1623,7 +1623,7 @@ } // These will be used by the masked-tx experiment in the future. -#if CONFIG_RECT_TX && CONFIG_EXT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) static void inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest, int stride, const TxfmParam *txfm_param) { #if CONFIG_LGT @@ -1659,7 +1659,7 @@ av1_iht32x8_256_add(input, dest, stride, txfm_param); #endif } -#endif // CONFIG_RECT_TX && CONFIG_EXT_TX && CONFIG_RECT_TX_EXT +#endif static void inv_txfm_add_8x16(const tran_low_t *input, uint8_t *dest, int stride, const TxfmParam *txfm_param) { @@ -2146,7 +2146,7 @@ #if CONFIG_CHROMA_2X2 case TX_2X2: inv_txfm_add_2x2(input, dest, stride, txfm_param); break; #endif -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) case TX_32X8: inv_txfm_add_32x8(input, dest, stride, txfm_param); break; case TX_8X32: inv_txfm_add_8x32(input, dest, stride, txfm_param); break; case TX_16X4: inv_txfm_add_16x4(input, dest, stride, txfm_param); break;
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 21e9add..80ef8e8 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -723,17 +723,33 @@ pd->dst.stride, max_scan_line, eob); *eob_total += eob; } else { +#if CONFIG_RECT_TX_EXT + int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize]; + const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size]; + if (is_qttx) assert(blk_row == 0 && blk_col == 0 && block == 0); +#else const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + assert(sub_txs < tx_size); +#endif const int bsl = tx_size_wide_unit[sub_txs]; int sub_step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; - assert(sub_txs < tx_size); int i; assert(bsl > 0); for (i = 0; i < 4; ++i) { +#if CONFIG_RECT_TX_EXT + int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs]; + const int offsetr = + is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0) + : blk_row + ((i >> 1) * bsl); + const int offsetc = + is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs]) + : blk_col + (i & 0x01) * bsl; +#else const int offsetr = blk_row + (i >> 1) * bsl; const int offsetc = blk_col + (i & 0x01) * bsl; +#endif if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; @@ -4961,10 +4977,10 @@ } #endif -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) if (cm->tx_mode == TX_MODE_SELECT) av1_diff_update_prob(&r, &fc->quarter_tx_size_prob, ACCT_STR); -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_LV_MAP av1_read_txb_probs(fc, cm->tx_mode, &r);
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index 0149688..7c85442 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -543,16 +543,22 @@ #if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX) if (coded_tx_size > max_txsize_lookup[bsize]) { assert(coded_tx_size == max_txsize_lookup[bsize] + 1); -#if CONFIG_EXT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT if (is_quarter_tx_allowed(xd, &xd->mi[0]->mbmi, is_inter)) { - int quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR); - FRAME_COUNTS *counts = xd->counts; + int quarter_tx; - if (counts) ++counts->quarter_tx_size[quarter_tx]; + if (quarter_txsize_lookup[bsize] != max_txsize_lookup[bsize]) { + quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR); + FRAME_COUNTS *counts = xd->counts; + + if (counts) ++counts->quarter_tx_size[quarter_tx]; + } else { + quarter_tx = 1; + } return quarter_tx ? quarter_txsize_lookup[bsize] : max_txsize_rect_lookup[bsize]; } -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX_EXT +#endif // CONFIG_RECT_TX_EXT return max_txsize_rect_lookup[bsize]; } @@ -2860,6 +2866,26 @@ for (idx = 0; idx < width; idx += bw) read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, height != width, idy, idx, r); +#if CONFIG_RECT_TX_EXT + if (is_quarter_tx_allowed(xd, mbmi, inter_block) && + mbmi->tx_size == max_tx_size) { + int quarter_tx; + + if (quarter_txsize_lookup[bsize] != max_tx_size) { + quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR); + if (xd->counts) ++xd->counts->quarter_tx_size[quarter_tx]; + } else { + quarter_tx = 1; + } + if (quarter_tx) { + mbmi->tx_size = quarter_txsize_lookup[bsize]; + for (idy = 0; idy < tx_size_high_unit[max_tx_size] / 2; ++idy) + for (idx = 0; idx < tx_size_wide_unit[max_tx_size] / 2; ++idx) + mbmi->inter_tx_size[idy][idx] = mbmi->tx_size; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); + } + } +#endif } else { mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r);
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 2331307..9241572 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -355,7 +355,12 @@ return; } +#if CONFIG_RECT_TX_EXT + if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] || + mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) { +#else if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) { +#endif #if CONFIG_NEW_MULTISYMBOL aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2); #else @@ -364,6 +369,7 @@ txfm_partition_update(xd->above_txfm_context + blk_col, xd->left_txfm_context + blk_row, tx_size, tx_size); + // TODO(yuec): set correct txfm partition update for qttx } else { const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; const int bsl = tx_size_wide_unit[sub_txs]; @@ -427,11 +433,11 @@ aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx], tx_size_cat + 2); -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size) aom_write(w, tx_size == quarter_txsize_lookup[bsize], cm->fc->quarter_tx_size_prob); -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif } } @@ -963,15 +969,30 @@ token_stats->cost += tmp_token_stats.cost; #endif } else { +#if CONFIG_RECT_TX_EXT + int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize]; + const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size]; +#else const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; +#endif const int bsl = tx_size_wide_unit[sub_txs]; int i; assert(bsl > 0); for (i = 0; i < 4; ++i) { +#if CONFIG_RECT_TX_EXT + int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs]; + const int offsetr = + is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0) + : blk_row + (i >> 1) * bsl; + const int offsetc = + is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs]) + : blk_col + (i & 0x01) * bsl; +#else const int offsetr = blk_row + (i >> 1) * bsl; const int offsetc = blk_col + (i & 0x01) * bsl; +#endif const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue; @@ -1748,6 +1769,15 @@ for (idx = 0; idx < width; idx += bw) write_tx_size_vartx(cm, xd, mbmi, max_tx_size, height != width, idy, idx, w); +#if CONFIG_RECT_TX_EXT + if (is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) && + quarter_txsize_lookup[bsize] != max_tx_size && + (mbmi->tx_size == quarter_txsize_lookup[bsize] || + mbmi->tx_size == max_tx_size)) { + aom_write(w, mbmi->tx_size != max_tx_size, + cm->fc->quarter_tx_size_prob); + } +#endif } else { set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, skip, xd); write_selected_tx_size(cm, xd, w); @@ -4551,11 +4581,11 @@ #if CONFIG_LOOP_RESTORATION encode_restoration(cm, header_bc); #endif // CONFIG_LOOP_RESTORATION -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) if (cm->tx_mode == TX_MODE_SELECT) av1_cond_prob_diff_update(header_bc, &cm->fc->quarter_tx_size_prob, cm->counts.quarter_tx_size, probwt); -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_LV_MAP av1_write_txb_probs(cpi, header_bc); #endif // CONFIG_LV_MAP
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index ab1218a..d13eb42 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -5474,10 +5474,15 @@ #endif // CONFIG_EXT_INTER #if CONFIG_VAR_TX +#if CONFIG_RECT_TX_EXT + if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0 && + counts->quarter_tx_size[1] == 0) +#else if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0) +#endif cm->tx_mode = ALLOW_32X32 + CONFIG_TX64X64; #else -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && CONFIG_EXT_TX if (cm->tx_mode == TX_MODE_SELECT && counts->quarter_tx_size[1] == 0) { #else if (cm->tx_mode == TX_MODE_SELECT) { @@ -5766,9 +5771,17 @@ if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; +#if CONFIG_RECT_TX_EXT + if (tx_size == plane_tx_size || + mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) { +#else if (tx_size == plane_tx_size) { +#endif ++counts->txfm_partition[ctx][0]; - mbmi->tx_size = tx_size; +#if CONFIG_RECT_TX_EXT + if (tx_size == plane_tx_size) +#endif + mbmi->tx_size = tx_size; txfm_partition_update(xd->above_txfm_context + blk_col, xd->left_txfm_context + blk_row, tx_size, tx_size); } else { @@ -6082,13 +6095,16 @@ ++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth]; #endif -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT + +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) if (is_quarter_tx_allowed(xd, mbmi, is_inter) && - mbmi->tx_size != txsize_sqr_up_map[mbmi->tx_size]) { - ++td->counts->quarter_tx_size[mbmi->tx_size == - quarter_txsize_lookup[mbmi->sb_type]]; + quarter_txsize_lookup[bsize] != max_txsize_rect_lookup[bsize] && + (mbmi->tx_size == quarter_txsize_lookup[bsize] || + mbmi->tx_size == max_txsize_rect_lookup[bsize])) { + ++td->counts + ->quarter_tx_size[mbmi->tx_size == quarter_txsize_lookup[bsize]]; } -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_EXT_TX && CONFIG_RECT_TX assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi))); #endif // CONFIG_EXT_TX && CONFIG_RECT_TX
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index 5b91cec..e7f4d31 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -797,16 +797,32 @@ encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); } else { assert(tx_size < TX_SIZES_ALL); +#if CONFIG_RECT_TX_EXT + int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize]; + const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size]; + if (is_qttx) assert(blk_row == 0 && blk_col == 0 && block == 0); +#else const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; assert(sub_txs < tx_size); +#endif // This is the square transform block partition entry point. int bsl = tx_size_wide_unit[sub_txs]; int i; assert(bsl > 0); for (i = 0; i < 4; ++i) { +#if CONFIG_RECT_TX_EXT + int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs]; + const int offsetr = + is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0) + : blk_row + ((i >> 1) * bsl); + const int offsetc = + is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs]) + : blk_col + ((i & 0x01) * bsl); +#else const int offsetr = blk_row + ((i >> 1) * bsl); const int offsetc = blk_col + ((i & 0x01) * bsl); +#endif int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs]; if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c index 81b895c..85f4b7d 100644 --- a/av1/encoder/hybrid_fwd_txfm.c +++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -143,7 +143,7 @@ } #endif // CONFIG_TX64X64 -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) static void fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff, int diff_stride, TxfmParam *txfm_param) { #if CONFIG_LGT @@ -179,7 +179,7 @@ av1_fht8x32(src_diff, coeff, diff_stride, txfm_param); #endif } -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif #if CONFIG_CHROMA_2X2 static void highbd_fwd_txfm_2x2(const int16_t *src_diff, tran_low_t *coeff, @@ -483,7 +483,7 @@ #if CONFIG_CHROMA_2X2 case TX_2X2: fwd_txfm_2x2(src_diff, coeff, diff_stride, txfm_param); break; #endif -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) case TX_4X16: fwd_txfm_4x16(src_diff, coeff, diff_stride, txfm_param); break; @@ -496,7 +496,7 @@ case TX_32X8: fwd_txfm_32x8(src_diff, coeff, diff_stride, txfm_param); break; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif default: assert(0); break; } }
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c index 9601184..da3b6e2 100644 --- a/av1/encoder/rd.c +++ b/av1/encoder/rd.c
@@ -669,7 +669,7 @@ for (i = 0; i < num_4x4_h; i += 8) t_left[i] = !!*(const uint64_t *)&left[i]; break; -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) case TX_4X16: for (i = 0; i < num_4x4_w; i += 2) t_above[i] = !!*(const uint16_t *)&above[i]; @@ -696,7 +696,7 @@ for (i = 0; i < num_4x4_h; i += 4) t_left[i] = !!*(const uint32_t *)&left[i]; break; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif default: assert(0 && "Invalid transform size."); break; } @@ -770,7 +770,7 @@ for (i = 0; i < num_4x4_h; i += 4) t_left[i] = !!*(const uint32_t *)&left[i]; break; -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) case TX_4X16: memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w); for (i = 0; i < num_4x4_h; i += 4) @@ -793,7 +793,7 @@ for (i = 0; i < num_4x4_h; i += 2) t_left[i] = !!*(const uint16_t *)&left[i]; break; -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif default: assert(0 && "Invalid transform size."); break; } }
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 17d882c..dc6837e 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -1556,9 +1556,9 @@ BLOCK_SIZE plane_bsize, int blk_row, int blk_col, BLOCK_SIZE tx_bsize, int *width, int *height, int *visible_width, int *visible_height) { -#if !(CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT) +#if !(CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)) assert(tx_bsize <= plane_bsize); -#endif // !(CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT) +#endif int txb_height = block_size_high[tx_bsize]; int txb_width = block_size_wide[tx_bsize]; const int block_height = block_size_high[plane_bsize]; @@ -1606,7 +1606,7 @@ x->qindex); #endif // CONFIG_DIST_8X8 -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) if ((txb_rows == visible_rows && txb_cols == visible_cols) && tx_bsize < BLOCK_SIZES) { #else @@ -2191,11 +2191,11 @@ const int depth = tx_size_to_depth(coded_tx_size); const int tx_size_ctx = get_tx_size_context(xd); int r_tx_size = cpi->tx_size_cost[tx_size_cat][tx_size_ctx][depth]; -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size) r_tx_size += av1_cost_bit(cm->fc->quarter_tx_size_prob, tx_size == quarter_txsize_lookup[bsize]); -#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#endif return r_tx_size; } else { return 0; @@ -4483,6 +4483,24 @@ TX_TYPE best_tx_type = TX_TYPES; int txk_idx = (blk_row << 4) + blk_col; #endif +#if CONFIG_RECT_TX_EXT + TX_SIZE quarter_txsize = quarter_txsize_lookup[mbmi->sb_type]; + int check_qttx = is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) && + tx_size == max_txsize_rect_lookup[mbmi->sb_type] && + quarter_txsize != tx_size; + int is_qttx_picked = 0; + int eobs_qttx[2] = { 0, 0 }; + int skip_qttx[2] = { 0, 0 }; + int block_offset_qttx = check_qttx + ? tx_size_wide_unit[quarter_txsize] * + tx_size_high_unit[quarter_txsize] + : 0; + int blk_row_offset, blk_col_offset; + int is_wide_qttx = + tx_size_wide_unit[quarter_txsize] > tx_size_high_unit[quarter_txsize]; + blk_row_offset = is_wide_qttx ? tx_size_high_unit[quarter_txsize] : 0; + blk_col_offset = is_wide_qttx ? 0 : tx_size_wide_unit[quarter_txsize]; +#endif av1_init_rd_stats(&sum_rd_stats); @@ -4504,7 +4522,7 @@ zero_blk_rate = av1_cost_bit(xd->fc->txb_skip[txs_ctx][txb_ctx.txb_skip_ctx], 1); #else - const int tx_size_ctx = txsize_sqr_map[tx_size]; + int tx_size_ctx = txsize_sqr_map[tx_size]; int coeff_ctx = get_entropy_context(tx_size, pta, ptl); zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0] [coeff_ctx][EOB_TOKEN]; @@ -4552,6 +4570,12 @@ if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) rd_stats->rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0); +#if CONFIG_RECT_TX_EXT + if (check_qttx) { + assert(blk_row == 0 && blk_col == 0); + rd_stats->rate += av1_cost_bit(cpi->common.fc->quarter_tx_size_prob, 0); + } +#endif this_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist); #if CONFIG_LV_MAP tmp_eob = p->txb_entropy_ctx[block]; @@ -4562,6 +4586,102 @@ #if CONFIG_TXK_SEL best_tx_type = mbmi->txk_type[txk_idx]; #endif + +#if CONFIG_RECT_TX_EXT + if (check_qttx) { + assert(blk_row == 0 && blk_col == 0 && block == 0 && plane == 0); + + RD_STATS rd_stats_tmp, rd_stats_qttx; + int64_t rd_qttx; + + av1_init_rd_stats(&rd_stats_qttx); + av1_init_rd_stats(&rd_stats_tmp); + + av1_tx_block_rd_b(cpi, x, quarter_txsize, 0, 0, plane, 0, plane_bsize, + pta, ptl, &rd_stats_qttx); + + tx_size_ctx = txsize_sqr_map[quarter_txsize]; + coeff_ctx = get_entropy_context(quarter_txsize, pta, ptl); + zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0] + [coeff_ctx][EOB_TOKEN]; + if ((RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist) >= + RDCOST(x->rdmult, zero_blk_rate, rd_stats_qttx.sse) || + rd_stats_qttx.skip == 1) && + !xd->lossless[mbmi->segment_id]) { +#if CONFIG_RD_DEBUG + av1_update_txb_coeff_cost(&rd_stats_qttx, plane, quarter_txsize, 0, 0, + zero_blk_rate - rd_stats_qttx.rate); +#endif // CONFIG_RD_DEBUG + rd_stats_qttx.rate = zero_blk_rate; + rd_stats_qttx.dist = rd_stats_qttx.sse; + rd_stats_qttx.skip = 1; + x->blk_skip[plane][blk_row * bw + blk_col] = 1; + skip_qttx[0] = 1; + p->eobs[block] = 0; + } else { + x->blk_skip[plane][blk_row * bw + blk_col] = 0; + skip_qttx[0] = 0; + rd_stats->skip = 0; + } + + // Second tx block + av1_tx_block_rd_b(cpi, x, quarter_txsize, blk_row_offset, blk_col_offset, + plane, block_offset_qttx, plane_bsize, pta, ptl, + &rd_stats_tmp); + + av1_set_txb_context(x, plane, 0, quarter_txsize, pta, ptl); + coeff_ctx = get_entropy_context(quarter_txsize, pta + blk_col_offset, + ptl + blk_row_offset); + zero_blk_rate = x->token_costs[tx_size_ctx][pd->plane_type][1][0][0] + [coeff_ctx][EOB_TOKEN]; + if ((RDCOST(x->rdmult, rd_stats_tmp.rate, rd_stats_tmp.dist) >= + RDCOST(x->rdmult, zero_blk_rate, rd_stats_tmp.sse) || + rd_stats_tmp.skip == 1) && + !xd->lossless[mbmi->segment_id]) { +#if CONFIG_RD_DEBUG + av1_update_txb_coeff_cost(&rd_stats_tmp, plane, quarter_txsize, 0, 0, + zero_blk_rate - rd_stats_tmp.rate); +#endif // CONFIG_RD_DEBUG + rd_stats_tmp.rate = zero_blk_rate; + rd_stats_tmp.dist = rd_stats_tmp.sse; + rd_stats_tmp.skip = 1; + x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = 1; + skip_qttx[1] = 1; + p->eobs[block_offset_qttx] = 0; + } else { + x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = 0; + skip_qttx[1] = 0; + rd_stats_tmp.skip = 0; + } + + av1_merge_rd_stats(&rd_stats_qttx, &rd_stats_tmp); + + if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) { + rd_stats_qttx.rate += + av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0); + } + rd_stats_qttx.rate += + av1_cost_bit(cpi->common.fc->quarter_tx_size_prob, 1); + rd_qttx = RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist); +#if CONFIG_LV_MAP + eobs_qttx[0] = p->txb_entropy_ctx[0]; + eobs_qttx[1] = p->txb_entropy_ctx[block_offset_qttx]; +#else + eobs_qttx[0] = p->eobs[0]; + eobs_qttx[1] = p->eobs[block_offset_qttx]; +#endif + if (rd_qttx < this_rd) { + is_qttx_picked = 1; + this_rd = rd_qttx; + rd_stats->rate = rd_stats_qttx.rate; + rd_stats->dist = rd_stats_qttx.dist; + rd_stats->sse = rd_stats_qttx.sse; + rd_stats->skip = rd_stats_qttx.skip; + rd_stats->rdcost = rd_stats_qttx.rdcost; + } + av1_get_entropy_contexts(plane_bsize, 0, pd, ta, tl); + } +#endif } #if CONFIG_MRC_TX @@ -4704,27 +4824,61 @@ if (this_rd < sum_rd) { int idx, idy; +#if CONFIG_RECT_TX_EXT + TX_SIZE tx_size_selected = is_qttx_picked ? quarter_txsize : tx_size; +#else + TX_SIZE tx_size_selected = tx_size; +#endif +#if CONFIG_RECT_TX_EXT + if (is_qttx_picked) { + assert(blk_row == 0 && blk_col == 0 && plane == 0); #if CONFIG_LV_MAP - p->txb_entropy_ctx[block] = tmp_eob; + p->txb_entropy_ctx[0] = eobs_qttx[0]; + p->txb_entropy_ctx[block_offset_qttx] = eobs_qttx[1]; +#else + p->eobs[0] = eobs_qttx[0]; + p->eobs[block_offset_qttx] = eobs_qttx[1]; +#endif + } else { +#endif +#if CONFIG_LV_MAP + p->txb_entropy_ctx[block] = tmp_eob; #else p->eobs[block] = tmp_eob; #endif +#if CONFIG_RECT_TX_EXT + } +#endif - av1_set_txb_context(x, plane, block, tx_size, pta, ptl); + av1_set_txb_context(x, plane, block, tx_size_selected, pta, ptl); +#if CONFIG_RECT_TX_EXT + if (is_qttx_picked) + av1_set_txb_context(x, plane, block_offset_qttx, tx_size_selected, + pta + blk_col_offset, ptl + blk_row_offset); +#endif txfm_partition_update(tx_above + blk_col, tx_left + blk_row, tx_size, tx_size); - inter_tx_size[0][0] = tx_size; + inter_tx_size[0][0] = tx_size_selected; for (idy = 0; idy < tx_size_high_unit[tx_size] / 2; ++idy) for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx) - inter_tx_size[idy][idx] = tx_size; - mbmi->tx_size = tx_size; + inter_tx_size[idy][idx] = tx_size_selected; + mbmi->tx_size = tx_size_selected; #if CONFIG_TXK_SEL mbmi->txk_type[txk_idx] = best_tx_type; #endif if (this_rd == INT64_MAX) *is_cost_valid = 0; - x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip; +#if CONFIG_RECT_TX_EXT + if (is_qttx_picked) { + x->blk_skip[plane][0] = skip_qttx[0]; + x->blk_skip[plane][blk_row_offset * bw + blk_col_offset] = skip_qttx[1]; + } else { +#endif + x->blk_skip[plane][blk_row * bw + blk_col] = rd_stats->skip; +#if CONFIG_RECT_TX_EXT + } +#endif } else { *rd_stats = sum_rd_stats; if (sum_rd == INT64_MAX) *is_cost_valid = 0;
diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c index 719799a..b9db891 100644 --- a/av1/encoder/tokenize.c +++ b/av1/encoder/tokenize.c
@@ -597,16 +597,31 @@ cost_coeffs_b(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); #endif } else { +#if CONFIG_RECT_TX_EXT + int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize]; + const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size]; +#else // Half the block size in transform block unit. const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; +#endif const int bsl = tx_size_wide_unit[sub_txs]; int i; assert(bsl > 0); for (i = 0; i < 4; ++i) { +#if CONFIG_RECT_TX_EXT + int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs]; + const int offsetr = + is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0) + : blk_row + ((i >> 1) * bsl); + const int offsetc = + is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs]) + : blk_col + ((i & 0x01) * bsl); +#else const int offsetr = blk_row + ((i >> 1) * bsl); const int offsetc = blk_col + ((i & 0x01) * bsl); +#endif int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
diff --git a/configure b/configure index c8e1f3e..0e8d933 100755 --- a/configure +++ b/configure
@@ -555,6 +555,7 @@ enabled ext_comp_refs && enable_feature var_refs enabled ext_comp_refs && disable_feature one_sided_compound enabled altref2 && enable_feature ext_refs + enabled rect_tx_ext && enable_feature rect_tx if ! enabled delta_q && enabled ext_delta_q; then log_echo "ext_delta_q requires delta_q, so disabling ext_delta_q"
diff --git a/tools/aom_entropy_optimizer.c b/tools/aom_entropy_optimizer.c index 65b7d29..b892cc1 100644 --- a/tools/aom_entropy_optimizer.c +++ b/tools/aom_entropy_optimizer.c
@@ -678,7 +678,7 @@ /* Transform size */ // TODO(yuec): av1_tx_size_tree has variable sizes, so needs special handling -#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT +#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX) cts_each_dim[0] = 2; optimize_entropy_table(&fc.quarter_tx_size[0], probsfile, 1, cts_each_dim, NULL, 1,