Refactor get_ext_tx_set_type()
-avoid unnecessary table lookup
-remove unused tx type sets
Change-Id: I693ea585982aae73b9c0a7f075dad2d84f8bee62
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 51cd796..cfe9e7d 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -646,7 +646,7 @@
// Number of transform types in each set type
static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
- 1, 2, 5, 7, 7, 10, 12, 16, 16,
+ 1, 2, 5, 7, 12, 16,
};
static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
@@ -654,23 +654,19 @@
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
- { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
- { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
- { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
};
-static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, int is_inter,
- int use_reduced_set) {
+static INLINE TxSetType av1_get_ext_tx_set_type(TX_SIZE tx_size, int is_inter,
+ int use_reduced_set) {
const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size];
- const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
-
if (tx_size_sqr_up > TX_32X32) return EXT_TX_SET_DCTONLY;
if (tx_size_sqr_up == TX_32X32)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
if (use_reduced_set)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
+ const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
if (is_inter) {
return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
: EXT_TX_SET_ALL16);
@@ -683,21 +679,22 @@
// Maps tx set types to the indices.
static const int ext_tx_set_index[2][EXT_TX_SET_TYPES] = {
{ // Intra
- 0, -1, 2, -1, 1, -1, -1, -1, -16 },
+ 0, -1, 2, 1, -1, -1 },
{ // Inter
- 0, 3, -1, -1, -1, -1, 2, -1, 1 },
+ 0, 3, -1, -1, 2, 1 },
};
static INLINE int get_ext_tx_set(TX_SIZE tx_size, int is_inter,
int use_reduced_set) {
const TxSetType set_type =
- get_ext_tx_set_type(tx_size, is_inter, use_reduced_set);
+ av1_get_ext_tx_set_type(tx_size, is_inter, use_reduced_set);
return ext_tx_set_index[is_inter][set_type];
}
static INLINE int get_ext_tx_types(TX_SIZE tx_size, int is_inter,
int use_reduced_set) {
- const int set_type = get_ext_tx_set_type(tx_size, is_inter, use_reduced_set);
+ const int set_type =
+ av1_get_ext_tx_set_type(tx_size, is_inter, use_reduced_set);
return av1_num_ext_tx_set[set_type];
}
@@ -824,7 +821,7 @@
const MB_MODE_INFO *const mbmi = xd->mi[0];
const struct macroblockd_plane *const pd = &xd->plane[plane_type];
const TxSetType tx_set_type =
- get_ext_tx_set_type(tx_size, is_inter_block(mbmi), reduced_tx_set);
+ av1_get_ext_tx_set_type(tx_size, is_inter_block(mbmi), reduced_tx_set);
TX_TYPE tx_type;
if (xd->lossless[mbmi->segment_id] || txsize_sqr_up_map[tx_size] > TX_32X32) {
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 292cd89..342025b 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -261,11 +261,8 @@
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 3, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 5, 6, 4, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0 },
- { 1, 5, 6, 4, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0 },
- { 1, 2, 3, 6, 4, 5, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0 },
{ 3, 4, 5, 8, 6, 7, 9, 10, 11, 0, 1, 2, 0, 0, 0, 0 },
{ 7, 8, 9, 12, 10, 11, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6 },
- { 7, 8, 9, 12, 10, 11, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6 },
};
static const int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES] = {
@@ -273,11 +270,8 @@
{ 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 9, 0, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 9, 0, 10, 11, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
- { 9, 0, 10, 11, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
- { 9, 0, 1, 2, 4, 5, 3, 6, 7, 8, 0, 0, 0, 0, 0, 0 },
{ 9, 10, 11, 0, 1, 2, 4, 5, 3, 6, 7, 8, 0, 0, 0, 0 },
{ 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 4, 5, 3, 6, 7, 8 },
- { 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 4, 5, 3, 6, 7, 8 },
};
void av1_set_default_ref_deltas(int8_t *ref_deltas);
diff --git a/av1/common/enums.h b/av1/common/enums.h
index c02f56f..72c631c 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -293,18 +293,10 @@
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
- // for 16x16 only
- EXT_TX_SET_DTT4_IDTX_1DDCT_16X16,
- // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT,
- // Discrete Trig transforms w/ flip (9) + Identity (1)
- EXT_TX_SET_DTT9_IDTX,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
- // for 16x16 only
- EXT_TX_SET_ALL16_16X16,
- // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16,
EXT_TX_SET_TYPES
} TxSetType;
diff --git a/av1/common/idct.c b/av1/common/idct.c
index 5d25043..7a2f3bc 100644
--- a/av1/common/idct.c
+++ b/av1/common/idct.c
@@ -326,7 +326,7 @@
txfm_param->lossless = xd->lossless[xd->mi[0]->segment_id];
txfm_param->bd = xd->bd;
txfm_param->is_hbd = get_bitdepth_data_path_index(xd);
- txfm_param->tx_set_type = get_ext_tx_set_type(
+ txfm_param->tx_set_type = av1_get_ext_tx_set_type(
txfm_param->tx_size, is_inter_block(xd->mi[0]), reduced_tx_set);
}
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 54b93eb..63c587a 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -624,7 +624,7 @@
!mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
const TxSetType tx_set_type =
- get_ext_tx_set_type(tx_size, inter_block, cm->reduced_tx_set_used);
+ av1_get_ext_tx_set_type(tx_size, inter_block, cm->reduced_tx_set_used);
const int eset =
get_ext_tx_set(tx_size, inter_block, cm->reduced_tx_set_used);
// eset == 0 should correspond to a set with only DCT_DCT and
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index efa5c29..f516fe6 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -803,7 +803,7 @@
!mbmi->skip &&
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
const TxSetType tx_set_type =
- get_ext_tx_set_type(tx_size, is_inter, cm->reduced_tx_set_used);
+ av1_get_ext_tx_set_type(tx_size, is_inter, cm->reduced_tx_set_used);
const int eset = get_ext_tx_set(tx_size, is_inter, cm->reduced_tx_set_used);
// eset == 0 should correspond to a set with only DCT_DCT and there
// is no need to send the tx_type
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index f403792..45e62db 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4604,7 +4604,7 @@
const int eset = get_ext_tx_set(tx_size, is_inter, cm->reduced_tx_set_used);
if (eset > 0) {
const TxSetType tx_set_type =
- get_ext_tx_set_type(tx_size, is_inter, cm->reduced_tx_set_used);
+ av1_get_ext_tx_set_type(tx_size, is_inter, cm->reduced_tx_set_used);
if (is_inter) {
if (allow_update_cdf) {
update_cdf(fc->inter_ext_tx_cdf[eset][txsize_sqr_map[tx_size]],
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index afb1f68..8cdfdd5 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -174,7 +174,7 @@
txfm_param.tx_type = tx_type;
txfm_param.tx_size = tx_size;
txfm_param.lossless = xd->lossless[mbmi->segment_id];
- txfm_param.tx_set_type = get_ext_tx_set_type(
+ txfm_param.tx_set_type = av1_get_ext_tx_set_type(
txfm_param.tx_size, is_inter_block(mbmi), cm->reduced_tx_set_used);
txfm_param.bd = xd->bd;
@@ -374,7 +374,7 @@
txfm_param.tx_size = tx_size;
txfm_param.eob = p->eobs[block];
txfm_param.lossless = xd->lossless[xd->mi[0]->segment_id];
- txfm_param.tx_set_type = get_ext_tx_set_type(
+ txfm_param.tx_set_type = av1_get_ext_tx_set_type(
txfm_param.tx_size, is_inter_block(xd->mi[0]), cm->reduced_tx_set_used);
if (txfm_param.is_hbd) {
av1_highbd_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride, &txfm_param);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 19df865..ae448ff 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2232,7 +2232,7 @@
uint8_t best_txb_ctx = 0;
const TxSetType tx_set_type =
- get_ext_tx_set_type(tx_size, is_inter, cm->reduced_tx_set_used);
+ av1_get_ext_tx_set_type(tx_size, is_inter, cm->reduced_tx_set_used);
int prune = 0;
const int do_prune = plane == 0 && !fast_tx_search && txk_end != DCT_DCT &&
!(!is_inter && x->use_default_intra_tx_type) &&
@@ -2749,7 +2749,7 @@
const int is_inter = is_inter_block(mbmi);
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode);
const TxSetType tx_set_type =
- get_ext_tx_set_type(mbmi->tx_size, is_inter, cm->reduced_tx_set_used);
+ av1_get_ext_tx_set_type(mbmi->tx_size, is_inter, cm->reduced_tx_set_used);
prune_tx(cpi, bs, x, xd, tx_set_type, 0);
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOM_PLANE_Y, bs,
mbmi->tx_size, cpi->sf.use_fast_coef_costing);
@@ -4670,7 +4670,7 @@
param.bd = xd->bd;
param.is_hbd = get_bitdepth_data_path_index(xd);
param.lossless = 0;
- param.tx_set_type = get_ext_tx_set_type(
+ param.tx_set_type = av1_get_ext_tx_set_type(
param.tx_size, is_inter_block(xd->mi[0]), reduced_tx_set);
const uint32_t ac_q = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd);
uint32_t max_quantized_coef = 0;
@@ -4743,7 +4743,7 @@
// Get the tx_size 1 level down
const TX_SIZE min_tx_size = sub_tx_size_map[max_txsize_rect_lookup[bsize]];
const TxSetType tx_set_type =
- get_ext_tx_set_type(min_tx_size, is_inter, cm->reduced_tx_set_used);
+ av1_get_ext_tx_set_type(min_tx_size, is_inter, cm->reduced_tx_set_used);
const int within_border =
mi_row >= xd->tile.mi_row_start &&
(mi_row + mi_size_high[bsize] < xd->tile.mi_row_end) &&
