Add macro to remove 1D transforms from TX_16X16
This produces a 0.03% drop in performance but reduces
the tx search space.
Change-Id: I27ae62403f059c4654e9237c39f15bed7f860c15
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 34e3f53..e25fb23 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -761,7 +761,7 @@
// Number of transform types in each set type
static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
- 1, 2, 5, 7, 12, 16,
+ 1, 2, 5, 7, 10, 12, 16,
};
static const int av1_ext_tx_set_idx_to_type[2][AOMMAX(EXT_TX_SETS_INTRA,
@@ -772,7 +772,12 @@
},
{
// Inter
- EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
+ EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16,
+#if USE_1D_16X16
+ EXT_TX_SET_DTT9_IDTX_1DDCT,
+#else
+ EXT_TX_SET_DTT9_IDTX,
+#endif // USE_1D_16X16
EXT_TX_SET_DCT_IDTX,
}
};
@@ -791,6 +796,9 @@
1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
},
{
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+ },
+ {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
},
{
@@ -823,8 +831,12 @@
if (tx_size_sqr_up == TX_32X32)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
if (is_inter)
+#if USE_1D_16X16
return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
: EXT_TX_SET_ALL16);
+#else
+ return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX : EXT_TX_SET_ALL16);
+#endif // USE_1D_16X16
else
return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
: EXT_TX_SET_DTT4_IDTX_1DDCT);
@@ -834,11 +846,17 @@
static const int ext_tx_set_index[2][EXT_TX_SET_TYPES] = {
{
// Intra
- 0, -1, 2, 1, -1, -1,
+ 0, -1, 2, 1, -1, -1, -1,
},
{
// Inter
- 0, 3, -1, -1, 2, 1,
+ 0, 3, -1, -1,
+#if USE_1D_16X16
+ -1, 2,
+#else
+ 2, -1,
+#endif // USE_1D_16X16
+ 1,
},
};
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 29fe72d..ca85aa0 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1790,6 +1790,7 @@
21733, 24241, 26749, 28253, 29758, 31263) },
{ AOM_CDF16(1280, 1453, 1626, 2277, 2929, 3580, 4232, 16717, 19225,
21733, 24241, 26749, 28253, 29758, 31263) } },
+#if USE_1D_16X16
{ { AOM_CDF12(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904, 28370,
29836, 31302) },
{ AOM_CDF12(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904, 28370,
@@ -1798,6 +1799,16 @@
29836, 31302) },
{ AOM_CDF12(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904, 28370,
29836, 31302) } },
+#else
+ { { AOM_CDF10(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904,
+ 28370) },
+ { AOM_CDF10(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904,
+ 28370) },
+ { AOM_CDF10(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904,
+ 28370) },
+ { AOM_CDF10(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904,
+ 28370) } },
+#endif // USE_1D_16X16
{ { AOM_CDF2(1536) },
{ AOM_CDF2(1536) },
{ AOM_CDF2(1536) },
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 8bcc6a5..bf738c8 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -415,6 +415,7 @@
[CDF_SIZE(INTRA_MODES)];
#endif
+#define USE_1D_16X16 1
static const int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES] = {
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -429,6 +430,9 @@
1, 5, 6, 4, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0,
},
{
+ 1, 2, 3, 6, 4, 5, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0,
+ },
+ {
3, 4, 5, 8, 6, 7, 9, 10, 11, 0, 1, 2, 0, 0, 0, 0,
},
{
@@ -450,6 +454,9 @@
9, 0, 10, 11, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
{
+ 9, 0, 1, 2, 4, 5, 3, 6, 7, 8, 0, 0, 0, 0, 0, 0,
+ },
+ {
9, 10, 11, 0, 1, 2, 4, 5, 3, 6, 7, 8, 0, 0, 0, 0,
},
{
diff --git a/av1/common/enums.h b/av1/common/enums.h
index 8b97cd3..d72901e 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -350,6 +350,8 @@
EXT_TX_SET_DTT4_IDTX,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT,
+ // Discrete Trig transforms w/ flip (9) + Identity (1)
+ EXT_TX_SET_DTT9_IDTX,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
