Reduce transform set for 16x32/32x16 Previously the transform type sets used for 16x32/32x16 transform sizes corresponded to 16x16 transforms. Now they correspond to 32x32 transforms with a reduced available set. BDRATE impact: 0.05% worse for lowres, 0.13% worse for midres Change-Id: I01d405d497c13837f5404f9e0caf8c571f3b26cf
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 9dacbfb..950e564 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -596,21 +596,72 @@ #define ALLOW_INTRA_EXT_TX 1 -static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 }; -static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 }; +typedef enum { + // DCT only + EXT_TX_SET_DCTONLY = 0, + // DCT + Identity only + EXT_TX_SET_DCT_IDTX = 1, + // Discrete Trig transforms w/o flip (4) + Identity (1) + EXT_TX_SET_DTT4_IDTX = 2, + // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2) + EXT_TX_SET_DTT4_IDTX_1DDCT = 3, + // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2) + EXT_TX_SET_DTT9_IDTX_1DDCT = 4, + // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6) + EXT_TX_SET_ALL16 = 5, + EXT_TX_SET_TYPES +} TxSetType; -static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, - int use_default) { - if (use_default) return is_inter ? 3 : 2; +// Number of transform types in each set type +static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 }; + +// Maps intra set index to the set type +static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = { + EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX +}; + +// Maps inter set index to the set type +static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = { + EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT, + EXT_TX_SET_DCT_IDTX +}; + +// Maps set types above to the indices used for intra +static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2, + 1, -1, -1 }; + +// Maps set types above to the indices used for inter +static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = { + 0, 3, -1, -1, 2, 1 +}; + +static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs, + int is_inter, int use_default) { + const TX_SIZE tx_size2 = txsize_sqr_up_map[tx_size]; + if (use_default) return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX; tx_size = txsize_sqr_map[tx_size]; #if CONFIG_CB4X4 (void)bs; - if (tx_size > TX_32X32) return 0; + if (tx_size > TX_32X32) return EXT_TX_SET_DCTONLY; #else - if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0; + if (tx_size > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY; #endif - if (tx_size == TX_32X32) return is_inter ? 3 : 0; - return (tx_size == TX_16X16 ? 2 : 1); + if (tx_size2 == TX_32X32) + return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY; + if (is_inter) + return (tx_size == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT + : EXT_TX_SET_ALL16); + else + return (tx_size == TX_16X16 ? EXT_TX_SET_DTT4_IDTX + : EXT_TX_SET_DTT4_IDTX_1DDCT); +} + +static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, + int use_default) { + const TxSetType set_type = + get_ext_tx_set_type(tx_size, bs, is_inter, use_default); + return is_inter ? ext_tx_set_index_inter[set_type] + : ext_tx_set_index_intra[set_type]; } static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = @@ -664,8 +715,8 @@ static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, int use_default) { - const int set = get_ext_tx_set(tx_size, bs, is_inter, use_default); - return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set]; + const int set_type = get_ext_tx_set_type(tx_size, bs, is_inter, use_default); + return num_ext_tx_set[set_type]; } #if CONFIG_RECT_TX
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 2fcbee3..d791ac8 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -4404,7 +4404,7 @@ if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { if (!use_inter_ext_tx_for_txsize[s][i]) continue; - for (j = 0; j < num_ext_tx_set_inter[s] - 1; ++j) + for (j = 0; j < num_ext_tx_set[ext_tx_set_type_inter[s]] - 1; ++j) av1_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j], ACCT_STR); } } @@ -4415,7 +4415,7 @@ for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { if (!use_intra_ext_tx_for_txsize[s][i]) continue; for (j = 0; j < INTRA_MODES; ++j) - for (k = 0; k < num_ext_tx_set_intra[s] - 1; ++k) + for (k = 0; k < num_ext_tx_set[ext_tx_set_type_intra[s]] - 1; ++k) av1_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k], ACCT_STR); }
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 6506dc0..4106860 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -644,16 +644,18 @@ if (!use_inter_ext_tx_for_txsize[s][i]) continue; savings += prob_diff_update_savings( av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i], - cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt); + cm->counts.inter_ext_tx[s][i], + num_ext_tx_set[ext_tx_set_type_inter[s]], probwt); } do_update = savings > savings_thresh; aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); if (do_update) { for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { if (!use_inter_ext_tx_for_txsize[s][i]) continue; - prob_diff_update( - av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i], - cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt, w); + prob_diff_update(av1_ext_tx_inter_tree[s], + cm->fc->inter_ext_tx_prob[s][i], + cm->counts.inter_ext_tx[s][i], + num_ext_tx_set[ext_tx_set_type_inter[s]], probwt, w); } } } @@ -666,7 +668,8 @@ for (j = 0; j < INTRA_MODES; ++j) savings += prob_diff_update_savings( av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j], - cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s], probwt); + cm->counts.intra_ext_tx[s][i][j], + num_ext_tx_set[ext_tx_set_type_intra[s]], probwt); } do_update = savings > savings_thresh; aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); @@ -677,7 +680,7 @@ prob_diff_update(av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j], cm->counts.intra_ext_tx[s][i][j], - num_ext_tx_set_intra[s], probwt, w); + num_ext_tx_set[ext_tx_set_type_intra[s]], probwt, w); } } }