Refactor the ext-tx experiment Use common structure for inter and intra tx type information when possible. Change-Id: I1fd3bc86033871ffbcc2b496a31dca00b7d64b31
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index c381e3a..a9a1f6a 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -933,75 +933,142 @@ #if CONFIG_EXT_TX #define ALLOW_INTRA_EXT_TX 1 -typedef enum { - // DCT only - EXT_TX_SET_DCTONLY = 0, - // DCT + Identity only - EXT_TX_SET_DCT_IDTX, +// Number of transform types in each set type +static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = { + 1, 2, #if CONFIG_MRC_TX - // DCT + MRC_DCT + 2, 3, +#endif // CONFIG_MRC_TX + 5, 7, 12, 16, +}; + +// Maps intra set index to the set type +static const int av1_ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = { + EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX, +#if CONFIG_MRC_TX EXT_TX_SET_MRC_DCT, - // DCT + MRC_DCT + IDTX +#endif // CONFIG_MRC_TX +}; + +// Maps inter set index to the set type +static const int av1_ext_tx_set_type_inter[EXT_TX_SETS_INTER] = { + EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, + EXT_TX_SET_DTT9_IDTX_1DDCT, EXT_TX_SET_DCT_IDTX, +#if CONFIG_MRC_TX EXT_TX_SET_MRC_DCT_IDTX, #endif // CONFIG_MRC_TX - // Discrete Trig transforms w/o flip (4) + Identity (1) - EXT_TX_SET_DTT4_IDTX, - // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2) - EXT_TX_SET_DTT4_IDTX_1DDCT, - // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2) - EXT_TX_SET_DTT9_IDTX_1DDCT, - // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6) - EXT_TX_SET_ALL16, - EXT_TX_SET_TYPES -} TxSetType; +}; +// Maps set types above to the indices used for intra +static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { + 0, -1, #if CONFIG_MRC_TX -// Number of transform types in each set type -static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { - 1, 2, 2, 3, 5, 7, 12, 16 + 3, -1, +#endif // CONFIG_MRC_TX + 2, 1, -1, -1, }; -// Maps intra set index to the set type -static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = { - EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX, - EXT_TX_SET_MRC_DCT -}; - -// Maps inter set index to the set type -static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = { - EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT, - EXT_TX_SET_DCT_IDTX, EXT_TX_SET_MRC_DCT_IDTX -}; - -// Maps set types above to the indices used for intra -static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 3, -1, - 2, 1, -1, -1 }; - -// Maps set types above to the indices used for inter -static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = { 0, 3, -1, 4, - -1, -1, 2, 1 }; -#else // CONFIG_MRC_TX -// Number of transform types in each set type -static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 }; - -// Maps intra set index to the set type -static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = { - EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX -}; - -// Maps inter set index to the set type -static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = { - EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT, - EXT_TX_SET_DCT_IDTX -}; - -// Maps set types above to the indices used for intra -static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2, - 1, -1, -1 }; - // Maps set types above to the indices used for inter static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = { - 0, 3, -1, -1, 2, 1 + 0, 3, +#if CONFIG_MRC_TX + -1, 4, +#endif // CONFIG_MRC_TX + -1, -1, 2, 1, +}; + +static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = + { +#if CONFIG_CHROMA_2X2 + { 1, 1, 1, 1, 1 }, // unused + { 0, 1, 1, 0, 0 }, + { 0, 0, 0, 1, 0 }, +#if CONFIG_MRC_TX + { 0, 0, 0, 0, 1 }, +#endif // CONFIG_MRC_TX +#else // CONFIG_CHROMA_2X2 + { 1, 1, 1, 1 }, // unused + { 1, 1, 0, 0 }, + { 0, 0, 1, 0 }, +#if CONFIG_MRC_TX + { 0, 0, 0, 1 }, +#endif // CONFIG_MRC_TX +#endif // CONFIG_CHROMA_2X2 + }; + +static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] = + { +#if CONFIG_CHROMA_2X2 + { 1, 1, 1, 1, 1 }, // unused + { 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 }, { 0, 0, 0, 0, 1 }, +#if CONFIG_MRC_TX + { 0, 0, 0, 0, 1 }, +#endif // CONFIG_MRC_TX +#else // CONFIG_CHROMA_2X2 + { 1, 1, 1, 1 }, // unused + { 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 }, +#if CONFIG_MRC_TX + { 0, 0, 0, 1 }, +#endif // CONFIG_MRC_TX +#endif // CONFIG_CHROMA_2X2 + }; + +// 1D Transforms used in inter set, this needs to be changed if +// ext_tx_used_inter is changed +static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = { + { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 }, +#if CONFIG_MRC_TX + { 1, 0, 0, 1 }, +#endif // CONFIG_MRC_TX +}; + +#if CONFIG_MRC_TX +static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = { + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + }, + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + }, + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, + }, + { + 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + }, +}; +#else // CONFIG_MRC_TX +static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = { + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }, + { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, + }, + { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + }, }; #endif // CONFIG_MRC_TX @@ -1043,124 +1110,11 @@ : ext_tx_set_index_intra[set_type]; } -#if CONFIG_MRC_TX -static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = - { -#if CONFIG_CHROMA_2X2 - { 1, 1, 1, 1, 1 }, // unused - { 0, 1, 1, 0, 0 }, - { 0, 0, 0, 1, 0 }, - { 0, 0, 0, 0, 1 }, -#else - { 1, 1, 1, 1 }, // unused - { 1, 1, 0, 0 }, - { 0, 0, 1, 0 }, - { 0, 0, 0, 1 }, -#endif // CONFIG_CHROMA_2X2 - }; - -static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] = - { -#if CONFIG_CHROMA_2X2 - { 1, 1, 1, 1, 1 }, // unused - { 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 }, - { 0, 0, 0, 0, 1 }, { 0, 0, 0, 0, 1 }, -#else - { 1, 1, 1, 1 }, // unused - { 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 }, { 0, 0, 0, 1 }, -#endif // CONFIG_CHROMA_2X2 - }; - -// Transform types used in each intra set -static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = { - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1 }, -}; - -// Numbers of transform types used in each intra set -static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5, 2 }; - -// Transform types used in each inter set -static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = { - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 }, - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1 }, -}; - -// Numbers of transform types used in each inter set -static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2, 3 }; - -// 1D Transforms used in inter set, this needs to be changed if -// ext_tx_used_inter is changed -static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = { - { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 }, { 1, 0, 0, 1 } -}; -#else // CONFIG_MRC_TX -static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = - { -#if CONFIG_CHROMA_2X2 - { 1, 1, 1, 1, 1 }, // unused - { 0, 1, 1, 0, 0 }, - { 0, 0, 0, 1, 0 }, -#else - { 1, 1, 1, 1 }, // unused - { 1, 1, 0, 0 }, - { 0, 0, 1, 0 }, -#endif // CONFIG_CHROMA_2X2 - }; - -static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] = - { -#if CONFIG_CHROMA_2X2 - { 1, 1, 1, 1, 1 }, // unused - { 0, 1, 1, 0, 0 }, - { 0, 0, 0, 1, 0 }, - { 0, 0, 0, 0, 1 }, -#else - { 1, 1, 1, 1 }, // unused - { 1, 1, 0, 0 }, - { 0, 0, 1, 0 }, - { 0, 0, 0, 1 }, -#endif // CONFIG_CHROMA_2X2 - }; - -// Transform types used in each intra set -static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = { - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 }, -}; - -// Numbers of transform types used in each intra set -static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 }; - -// Transform types used in each inter set -static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = { - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 }, -}; - -// Numbers of transform types used in each inter set -static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 }; - -// 1D Transforms used in inter set, this needs to be changed if -// ext_tx_used_inter is changed -static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = { - { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 }, -}; -#endif // CONFIG_MRC_TX - static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, int use_reduced_set) { const int set_type = get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set); - return num_ext_tx_set[set_type]; + return av1_num_ext_tx_set[set_type]; } #if CONFIG_RECT_TX
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c index 05bf08c..f30f3fd 100644 --- a/av1/common/entropymode.c +++ b/av1/common/entropymode.c
@@ -1409,10 +1409,8 @@ #endif #endif #if CONFIG_EXT_TX -int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES]; -int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES]; -int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES]; -int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES]; +int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES]; +int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES]; #endif #if CONFIG_SMOOTH_HV @@ -2485,70 +2483,59 @@ #if CONFIG_EXT_TX /* clang-format off */ -const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER] - [TREE_SIZE(TX_TYPES)] = { - { // ToDo(yaowu): remove used entry 0. - 0 - }, { - -IDTX, 2, - 4, 14, - 6, 8, - -V_DCT, -H_DCT, - 10, 12, - -V_ADST, -H_ADST, - -V_FLIPADST, -H_FLIPADST, - -DCT_DCT, 16, - 18, 24, - 20, 22, - -ADST_DCT, -DCT_ADST, - -FLIPADST_DCT, -DCT_FLIPADST, - 26, 28, - -ADST_ADST, -FLIPADST_FLIPADST, - -ADST_FLIPADST, -FLIPADST_ADST - }, { - -IDTX, 2, - 4, 6, - -V_DCT, -H_DCT, - -DCT_DCT, 8, - 10, 16, - 12, 14, - -ADST_DCT, -DCT_ADST, - -FLIPADST_DCT, -DCT_FLIPADST, - 18, 20, - -ADST_ADST, -FLIPADST_FLIPADST, - -ADST_FLIPADST, -FLIPADST_ADST - }, { - -IDTX, -DCT_DCT, - }, +const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES][TREE_SIZE(TX_TYPES)] = { + // TODO(yaowu@google.com): remove used entry 0. + { 0 }, + { -IDTX, -DCT_DCT, }, #if CONFIG_MRC_TX - { - -IDTX, 2, -DCT_DCT, -MRC_DCT, - } + { -DCT_DCT, -MRC_DCT, }, + { -IDTX, 2, + -DCT_DCT, -MRC_DCT, }, #endif // CONFIG_MRC_TX -}; - -const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA] - [TREE_SIZE(TX_TYPES)] = { - { // ToDo(yaowu): remove unused entry 0. - 0 - }, { - -IDTX, 2, - -DCT_DCT, 4, - 6, 8, - -V_DCT, -H_DCT, - -ADST_ADST, 10, - -ADST_DCT, -DCT_ADST, - }, { - -IDTX, 2, - -DCT_DCT, 4, - -ADST_ADST, 6, - -ADST_DCT, -DCT_ADST, - }, -#if CONFIG_MRC_TX - { - -DCT_DCT, -MRC_DCT, - } -#endif // CONFIG_MRC_TX + { + -IDTX, 2, + -DCT_DCT, 4, + -ADST_ADST, 6, + -ADST_DCT, -DCT_ADST, + }, + { + -IDTX, 2, + -DCT_DCT, 4, + 6, 8, + -V_DCT, -H_DCT, + -ADST_ADST, 10, + -ADST_DCT, -DCT_ADST, + }, + { + -IDTX, 2, + 4, 6, + -V_DCT, -H_DCT, + -DCT_DCT, 8, + 10, 16, + 12, 14, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 18, 20, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST, + }, + { + -IDTX, 2, + 4, 14, + 6, 8, + -V_DCT, -H_DCT, + 10, 12, + -V_ADST, -H_ADST, + -V_FLIPADST, -H_FLIPADST, + -DCT_DCT, 16, + 18, 24, + 20, 22, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 26, 28, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST, + }, }; /* clang-format on */ @@ -5531,17 +5518,20 @@ int s; for (s = 1; s < EXT_TX_SETS_INTER; ++s) { if (use_inter_ext_tx_for_txsize[s][i]) { - aom_tree_merge_probs( - av1_ext_tx_inter_tree[s], pre_fc->inter_ext_tx_prob[s][i], - counts->inter_ext_tx[s][i], fc->inter_ext_tx_prob[s][i]); + aom_tree_merge_probs(av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]], + pre_fc->inter_ext_tx_prob[s][i], + counts->inter_ext_tx[s][i], + fc->inter_ext_tx_prob[s][i]); } } for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { if (use_intra_ext_tx_for_txsize[s][i]) { - for (j = 0; j < INTRA_MODES; ++j) - aom_tree_merge_probs( - av1_ext_tx_intra_tree[s], pre_fc->intra_ext_tx_prob[s][i][j], - counts->intra_ext_tx[s][i][j], fc->intra_ext_tx_prob[s][i][j]); + for (j = 0; j < INTRA_MODES; ++j) { + aom_tree_merge_probs(av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]], + pre_fc->intra_ext_tx_prob[s][i][j], + counts->intra_ext_tx[s][i][j], + fc->intra_ext_tx_prob[s][i][j]); + } } } }
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h index 05440e8..eb0dfa9 100644 --- a/av1/common/entropymode.h +++ b/av1/common/entropymode.h
@@ -551,10 +551,8 @@ extern const int av1_intra_mode_ind[INTRA_MODES]; extern const int av1_intra_mode_inv[INTRA_MODES]; #if CONFIG_EXT_TX -extern int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES]; -extern int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES]; -extern int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES]; -extern int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES]; +extern int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES]; +extern int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES]; #endif #if CONFIG_EXT_INTER @@ -588,6 +586,8 @@ [TREE_SIZE(TX_TYPES)]; extern const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA] [TREE_SIZE(TX_TYPES)]; +extern const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES] + [TREE_SIZE(TX_TYPES)]; #else extern const aom_tree_index av1_ext_tx_tree[TREE_SIZE(TX_TYPES)]; #endif // CONFIG_EXT_TX
diff --git a/av1/common/enums.h b/av1/common/enums.h index 4465735..6857818 100644 --- a/av1/common/enums.h +++ b/av1/common/enums.h
@@ -292,6 +292,28 @@ } TX_TYPE; #if CONFIG_EXT_TX +typedef enum { + // DCT only + EXT_TX_SET_DCTONLY = 0, + // DCT + Identity only + EXT_TX_SET_DCT_IDTX, +#if CONFIG_MRC_TX + // DCT + MRC_DCT + EXT_TX_SET_MRC_DCT, + // DCT + MRC_DCT + IDTX + EXT_TX_SET_MRC_DCT_IDTX, +#endif // CONFIG_MRC_TX + // Discrete Trig transforms w/o flip (4) + Identity (1) + EXT_TX_SET_DTT4_IDTX, + // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2) + EXT_TX_SET_DTT4_IDTX_1DDCT, + // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2) + EXT_TX_SET_DTT9_IDTX_1DDCT, + // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6) + EXT_TX_SET_ALL16, + EXT_TX_SET_TYPES +} TxSetType; + #define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX) #else #define IS_2D_TRANSFORM(tx_type) 1
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 00501eb..b19332a 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -2468,10 +2468,12 @@ const int eset = get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used); if (eset > 0) { + const TxSetType tx_set_type = get_ext_tx_set_type( + supertx_size, bsize, 1, cm->reduced_tx_set_used); const int packed_sym = aom_read_symbol(r, ec_ctx->inter_ext_tx_cdf[eset][supertx_size], - ext_tx_cnt_inter[eset], ACCT_STR); - txfm = av1_ext_tx_inter_inv[eset][packed_sym]; + av1_num_ext_tx_set[tx_set_type], ACCT_STR); + txfm = av1_ext_tx_inv[tx_set_type][packed_sym]; if (xd->counts) ++xd->counts->inter_ext_tx[eset][supertx_size][txfm]; } }
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index dc8eff7..f293f81 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -985,6 +985,8 @@ !supertx_enabled && #endif // CONFIG_SUPERTX !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + const TxSetType tx_set_type = get_ext_tx_set_type( + tx_size, mbmi->sb_type, inter_block, cm->reduced_tx_set_used); const int eset = get_ext_tx_set(tx_size, mbmi->sb_type, inter_block, cm->reduced_tx_set_used); // eset == 0 should correspond to a set with only DCT_DCT and @@ -993,14 +995,14 @@ FRAME_COUNTS *counts = xd->counts; if (inter_block) { - *tx_type = av1_ext_tx_inter_inv[eset][aom_read_symbol( + *tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol( r, ec_ctx->inter_ext_tx_cdf[eset][square_tx_size], - ext_tx_cnt_inter[eset], ACCT_STR)]; + av1_num_ext_tx_set[tx_set_type], ACCT_STR)]; if (counts) ++counts->inter_ext_tx[eset][square_tx_size][*tx_type]; } else if (ALLOW_INTRA_EXT_TX) { - *tx_type = av1_ext_tx_intra_inv[eset][aom_read_symbol( + *tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol( r, ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode], - ext_tx_cnt_intra[eset], ACCT_STR)]; + av1_num_ext_tx_set[tx_set_type], ACCT_STR)]; if (counts) ++counts->intra_ext_tx[eset][square_tx_size][mbmi->mode][*tx_type]; }
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c index d983a0f..ded933e 100644 --- a/av1/decoder/decoder.c +++ b/av1/decoder/decoder.c
@@ -55,13 +55,10 @@ av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv, av1_switchable_interp_tree); #if CONFIG_EXT_TX - int s; - for (s = 1; s < EXT_TX_SETS_INTRA; ++s) - av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s], - av1_ext_tx_intra_tree[s]); - for (s = 1; s < EXT_TX_SETS_INTER; ++s) - av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s], - av1_ext_tx_inter_tree[s]); + for (int s = 1; s < EXT_TX_SET_TYPES; ++s) { + av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s], + av1_ext_tx_tree[s]); + } #else av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree); #endif
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 88a31bb..0dcf735 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -121,16 +121,24 @@ void av1_encode_token_init(void) { #if CONFIG_EXT_TX int s; -#endif // CONFIG_EXT_TX -#if CONFIG_EXT_TX for (s = 1; s < EXT_TX_SETS_INTER; ++s) { - av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]); + av1_tokens_from_tree(ext_tx_inter_encodings[s], + av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]]); } for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { - av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]); + av1_tokens_from_tree(ext_tx_intra_encodings[s], + av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]]); + } + for (s = 1; s < EXT_TX_SET_TYPES; ++s) { + av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s], + av1_ext_tx_tree[s]); } #else av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree); + /* This hack is necessary because the four TX_TYPES are not consecutive, + e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree + structure. */ + av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree); #endif // CONFIG_EXT_TX #if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP @@ -158,19 +166,6 @@ an in-order traversal of the av1_switchable_interp_tree structure. */ av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv, av1_switchable_interp_tree); -/* This hack is necessary because the four TX_TYPES are not consecutive, - e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree - structure. */ -#if CONFIG_EXT_TX - for (s = 1; s < EXT_TX_SETS_INTRA; ++s) - av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s], - av1_ext_tx_intra_tree[s]); - for (s = 1; s < EXT_TX_SETS_INTER; ++s) - av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s], - av1_ext_tx_inter_tree[s]); -#else - av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree); -#endif } static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx, @@ -1607,23 +1602,23 @@ if (tx_type == MRC_DCT) assert(mbmi->valid_mrc_mask && "Invalid MRC mask"); #endif // CONFIG_MRC_TX - + const TxSetType tx_set_type = get_ext_tx_set_type( + tx_size, bsize, is_inter, cm->reduced_tx_set_used); const int eset = get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); // eset == 0 should correspond to a set with only DCT_DCT and there // is no need to send the tx_type assert(eset > 0); + assert(av1_ext_tx_used[tx_set_type][tx_type]); if (is_inter) { - assert(ext_tx_used_inter[eset][tx_type]); - aom_write_symbol(w, av1_ext_tx_inter_ind[eset][tx_type], + aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][tx_type], ec_ctx->inter_ext_tx_cdf[eset][square_tx_size], - ext_tx_cnt_inter[eset]); + av1_num_ext_tx_set[tx_set_type]); } else if (ALLOW_INTRA_EXT_TX) { - assert(ext_tx_used_intra[eset][tx_type]); aom_write_symbol( - w, av1_ext_tx_intra_ind[eset][tx_type], + w, av1_ext_tx_ind[tx_set_type][tx_type], ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode], - ext_tx_cnt_intra[eset]); + av1_num_ext_tx_set[tx_set_type]); } } #else @@ -3082,10 +3077,12 @@ !skip) { const int eset = get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used); + const int tx_set_type = + get_ext_tx_set_type(supertx_size, bsize, 1, cm->reduced_tx_set_used); if (eset > 0) { - aom_write_symbol(w, av1_ext_tx_inter_ind[eset][mbmi->tx_type], + aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][mbmi->tx_type], ec_ctx->inter_ext_tx_cdf[eset][supertx_size], - ext_tx_cnt_inter[eset]); + av1_num_ext_tx_set[tx_set_type]); } } #else
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 201f858..40f0064 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -6073,16 +6073,20 @@ const int eset = get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); if (eset > 0) { + const TxSetType tx_set_type = get_ext_tx_set_type( + tx_size, bsize, is_inter, cm->reduced_tx_set_used); if (is_inter) { update_cdf(fc->inter_ext_tx_cdf[eset][txsize_sqr_map[tx_size]], - av1_ext_tx_inter_ind[eset][tx_type], ext_tx_cnt_inter[eset]); + av1_ext_tx_ind[tx_set_type][tx_type], + av1_num_ext_tx_set[tx_set_type]); ++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type]; } else { ++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode] [tx_type]; update_cdf( fc->intra_ext_tx_cdf[eset][txsize_sqr_map[tx_size]][mbmi->mode], - av1_ext_tx_intra_ind[eset][tx_type], ext_tx_cnt_intra[eset]); + av1_ext_tx_ind[tx_set_type][tx_type], + av1_num_ext_tx_set[tx_set_type]); } } } @@ -7327,9 +7331,6 @@ TX_SIZE tx_size; MB_MODE_INFO *mbmi; TX_TYPE tx_type, best_tx_nostx; -#if CONFIG_EXT_TX - int ext_tx_set; -#endif // CONFIG_EXT_TX int tmp_rate_tx = 0, skip_tx = 0; int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX; @@ -7399,7 +7400,9 @@ tx_size = max_txsize_lookup[bsize]; av1_subtract_plane(x, bsize, 0); #if CONFIG_EXT_TX - ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used); + int ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used); + const TxSetType tx_set_type = + get_ext_tx_set_type(tx_size, bsize, 1, cm->reduced_tx_set_used); #endif // CONFIG_EXT_TX for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { #if CONFIG_VAR_TX @@ -7410,7 +7413,7 @@ #endif // CONFIG_VAR_TX #if CONFIG_EXT_TX - if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; + if (!av1_ext_tx_used[tx_set_type][tx_type]) continue; #else if (tx_size >= TX_32X32 && tx_type != DCT_DCT) continue; #endif // CONFIG_EXT_TX
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c index 1300578..b06c43d 100644 --- a/av1/encoder/encodetxb.c +++ b/av1/encoder/encodetxb.c
@@ -2044,12 +2044,11 @@ } #if CONFIG_EXT_TX - int is_inter = is_inter_block(mbmi); - int ext_tx_set = get_ext_tx_set(get_min_tx_size(tx_size), mbmi->sb_type, - is_inter, cm->reduced_tx_set_used); - if (!(is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) && - !(!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) - continue; + const int is_inter = is_inter_block(mbmi); + const TxSetType tx_set_type = + get_ext_tx_set_type(get_min_tx_size(tx_size), mbmi->sb_type, is_inter, + cm->reduced_tx_set_used); + if (!av1_ext_tx_used[tx_set_type][tx_type]) continue; #endif // CONFIG_EXT_TX RD_STATS this_rd_stats;
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c index 8a6ce0b..fd8b96e 100644 --- a/av1/encoder/rd.c +++ b/av1/encoder/rd.c
@@ -179,15 +179,16 @@ if (use_inter_ext_tx_for_txsize[s][i]) { av1_cost_tokens_from_cdf(x->inter_tx_type_costs[s][i], fc->inter_ext_tx_cdf[s][i], - av1_ext_tx_inter_inv[s]); + av1_ext_tx_inv[av1_ext_tx_set_type_inter[s]]); } } for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { if (use_intra_ext_tx_for_txsize[s][i]) { - for (j = 0; j < INTRA_MODES; ++j) - av1_cost_tokens_from_cdf(x->intra_tx_type_costs[s][i][j], - fc->intra_ext_tx_cdf[s][i][j], - av1_ext_tx_intra_inv[s]); + for (j = 0; j < INTRA_MODES; ++j) { + av1_cost_tokens_from_cdf( + x->intra_tx_type_costs[s][i][j], fc->intra_ext_tx_cdf[s][i][j], + av1_ext_tx_inv[av1_ext_tx_set_type_intra[s]]); + } } } }
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 019e1f5..cabfd7c 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -2311,7 +2311,7 @@ } } -// #TODO(angiebird): use this function whenever it's possible +// TODO(angiebird): use this function whenever it's possible int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane, TX_SIZE tx_size, TX_TYPE tx_type) { @@ -2442,10 +2442,10 @@ if (max_tx_size >= TX_32X32 && tx_size == TX_4X4) return 1; #if CONFIG_EXT_TX const AV1_COMMON *const cm = &cpi->common; - int ext_tx_set = - get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used); + const TxSetType tx_set_type = + get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used); + if (!av1_ext_tx_used[tx_set_type][tx_type]) return 1; if (is_inter) { - if (!ext_tx_used_inter[ext_tx_set][tx_type]) return 1; if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) { if (!do_tx_type_search(tx_type, prune)) return 1; } @@ -2453,7 +2453,6 @@ if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) { if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) return 1; } - if (!ext_tx_used_intra[ext_tx_set][tx_type]) return 1; } #else // CONFIG_EXT_TX if (tx_size >= TX_32X32 && tx_type != DCT_DCT) return 1; @@ -2494,9 +2493,6 @@ const int is_inter = is_inter_block(mbmi); int prune = 0; const int plane = 0; -#if CONFIG_EXT_TX - int ext_tx_set; -#endif // CONFIG_EXT_TX av1_invalid_rd_stats(rd_stats); mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter); @@ -2504,8 +2500,10 @@ mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); #endif // CONFIG_VAR_TX #if CONFIG_EXT_TX - ext_tx_set = + int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used); + const TxSetType tx_set_type = + get_ext_tx_set_type(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used); #endif // CONFIG_EXT_TX if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE) @@ -2526,12 +2524,12 @@ #endif // CONFIG_PVQ for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { + if (!av1_ext_tx_used[tx_set_type][tx_type]) continue; RD_STATS this_rd_stats; if (is_inter) { if (x->use_default_inter_tx_type && tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size)) continue; - if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) { if (!do_tx_type_search(tx_type, prune)) continue; } @@ -2542,7 +2540,6 @@ if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) { if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue; } - if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue; } mbmi->tx_type = tx_type; @@ -2696,10 +2693,9 @@ if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue; const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs]; RD_STATS this_rd_stats; - int ext_tx_set = - get_ext_tx_set(rect_tx_size, bs, is_inter, cm->reduced_tx_set_used); - if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) || - (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) { + const TxSetType tx_set_type = get_ext_tx_set_type( + rect_tx_size, bs, is_inter, cm->reduced_tx_set_used); + if (av1_ext_tx_used[tx_set_type][tx_type]) { rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, rect_tx_size); ref_best_rd = AOMMIN(rd, ref_best_rd); @@ -2745,10 +2741,9 @@ if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue; const TX_SIZE tx_size = quarter_txsize_lookup[bs]; RD_STATS this_rd_stats; - int ext_tx_set = - get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used); - if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) || - (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) { + const TxSetType tx_set_type = + get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used); + if (av1_ext_tx_used[tx_set_type][tx_type]) { rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, tx_size); if (rd < best_rd) { @@ -5271,6 +5266,8 @@ RD_STATS rd_stats_stack[4]; #endif // CONFIG_EXT_PARTITION #if CONFIG_EXT_TX + const TxSetType tx_set_type = get_ext_tx_set_type( + max_tx_size, bsize, is_inter, cm->reduced_tx_set_used); const int ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter, cm->reduced_tx_set_used); #endif // CONFIG_EXT_TX @@ -5315,8 +5312,8 @@ continue; #endif // CONFIG_MRC_TX #if CONFIG_EXT_TX + if (!av1_ext_tx_used[tx_set_type][tx_type]) continue; if (is_inter) { - if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) { if (!do_tx_type_search(tx_type, prune)) continue; } @@ -5324,7 +5321,6 @@ if (!ALLOW_INTRA_EXT_TX && bsize >= BLOCK_8X8) { if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue; } - if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue; } #else // CONFIG_EXT_TX if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE &&