Misc fixes for 32x64 and 64x32 transforms
Change-Id: Ic843e99bd9b79cb9a0a26b95e3a48717ff2ec2a5
diff --git a/aom_dsp/txfm_common.h b/aom_dsp/txfm_common.h
index c33e38e..2bf0403 100644
--- a/aom_dsp/txfm_common.h
+++ b/aom_dsp/txfm_common.h
@@ -91,6 +91,7 @@
// 16384 * sqrt(2)
static const tran_high_t Sqrt2 = 23170;
+static const tran_high_t InvSqrt2 = 11585;
static INLINE tran_high_t fdct_round_shift(tran_high_t input) {
tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
diff --git a/av1/common/av1_fwd_txfm1d.c b/av1/common/av1_fwd_txfm1d.c
index cfe2741..c9c7f43 100644
--- a/av1/common/av1_fwd_txfm1d.c
+++ b/av1/common/av1_fwd_txfm1d.c
@@ -1547,6 +1547,16 @@
for (int i = 0; i < 32; ++i) output[i] = input[i] * 4;
range_check(0, input, output, 32, stage_range[0]);
}
+
+#if CONFIG_TX64X64
+void av1_fidentity64_c(const int32_t *input, int32_t *output,
+ const int8_t *cos_bit, const int8_t *stage_range) {
+ (void)cos_bit;
+ for (int i = 0; i < 64; ++i)
+ output[i] = (int32_t)dct_const_round_shift(input[i] * 4 * Sqrt2);
+ range_check(0, input, output, 64, stage_range[0]);
+}
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
#if CONFIG_TX64X64
diff --git a/av1/common/av1_fwd_txfm1d.h b/av1/common/av1_fwd_txfm1d.h
index f641930..f880239 100644
--- a/av1/common/av1_fwd_txfm1d.h
+++ b/av1/common/av1_fwd_txfm1d.h
@@ -26,8 +26,10 @@
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fdct32_new(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
+#if CONFIG_TX64X64
void av1_fdct64_new(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
+#endif // CONFIG_TX64X64
void av1_fadst4_new(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
@@ -46,6 +48,10 @@
const int8_t *cos_bit, const int8_t *stage_range);
void av1_fidentity32_c(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
+#if CONFIG_TX64X64
+void av1_fidentity64_c(const int32_t *input, int32_t *output,
+ const int8_t *cos_bit, const int8_t *stage_range);
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
#ifdef __cplusplus
diff --git a/av1/common/av1_fwd_txfm2d.c b/av1/common/av1_fwd_txfm2d.c
index 650b5bd..d4ff86b 100644
--- a/av1/common/av1_fwd_txfm2d.c
+++ b/av1/common/av1_fwd_txfm2d.c
@@ -36,6 +36,9 @@
case TXFM_TYPE_IDENTITY8: return av1_fidentity8_c;
case TXFM_TYPE_IDENTITY16: return av1_fidentity16_c;
case TXFM_TYPE_IDENTITY32: return av1_fidentity32_c;
+#if CONFIG_TX64X64
+ case TXFM_TYPE_IDENTITY64: return av1_fidentity64_c;
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
default: assert(0); return NULL;
}
diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c
index 3bd8686..51f4b63 100644
--- a/av1/common/av1_inv_txfm1d.c
+++ b/av1/common/av1_inv_txfm1d.c
@@ -1593,6 +1593,16 @@
for (int i = 0; i < 32; ++i) output[i] = input[i] * 4;
range_check(0, input, output, 32, stage_range[0]);
}
+
+#if CONFIG_TX64X64
+void av1_iidentity64_c(const int32_t *input, int32_t *output,
+ const int8_t *cos_bit, const int8_t *stage_range) {
+ (void)cos_bit;
+ for (int i = 0; i < 64; ++i)
+ output[i] = (int32_t)dct_const_round_shift(input[i] * 4 * Sqrt2);
+ range_check(0, input, output, 64, stage_range[0]);
+}
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
#if CONFIG_TX64X64
diff --git a/av1/common/av1_inv_txfm1d.h b/av1/common/av1_inv_txfm1d.h
index 037a3c6..8996f7c 100644
--- a/av1/common/av1_inv_txfm1d.h
+++ b/av1/common/av1_inv_txfm1d.h
@@ -26,8 +26,10 @@
const int8_t *cos_bit, const int8_t *stage_range);
void av1_idct32_new(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
+#if CONFIG_TX64X64
void av1_idct64_new(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
+#endif // CONFIG_TX64X64
void av1_iadst4_new(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
@@ -46,6 +48,10 @@
const int8_t *cos_bit, const int8_t *stage_range);
void av1_iidentity32_c(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range);
+#if CONFIG_TX64X64
+void av1_iidentity64_c(const int32_t *input, int32_t *output,
+ const int8_t *cos_bit, const int8_t *stage_range);
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
#ifdef __cplusplus
diff --git a/av1/common/av1_inv_txfm1d_cfg.h b/av1/common/av1_inv_txfm1d_cfg.h
index 78c9585..aeb2aec 100644
--- a/av1/common/av1_inv_txfm1d_cfg.h
+++ b/av1/common/av1_inv_txfm1d_cfg.h
@@ -173,6 +173,7 @@
TXFM_TYPE_DCT32 // .txfm_type
};
+#if CONFIG_TX64X64
// ---------------- row config inv_dct_64 ----------------
static const TXFM_1D_CFG inv_txfm_1d_row_cfg_dct_64 = {
64, // .txfm_size
@@ -182,6 +183,7 @@
inv_cos_bit_row_dct_64, // .cos_bit
TXFM_TYPE_DCT64, // .txfm_type_col
};
+#endif // CONFIG_TX64X64
// ---------------- row config inv_adst_4 ----------------
static const TXFM_1D_CFG inv_txfm_1d_row_cfg_adst_4 = {
@@ -353,5 +355,17 @@
NULL, // .cos_bit
TXFM_TYPE_IDENTITY32, // .txfm_type
};
+
+#if CONFIG_TX64X64
+// ---------------- row/col config inv_identity_32 ----------------
+static const TXFM_1D_CFG inv_txfm_1d_cfg_identity_64 = {
+ 64, // .txfm_size
+ 1, // .stage_num
+ inv_shift_64, // .shift
+ inv_stage_range_idx_64, // .stage_range
+ NULL, // .cos_bit
+ TXFM_TYPE_IDENTITY64, // .txfm_type
+};
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
#endif // AV1_INV_TXFM2D_CFG_H_
diff --git a/av1/common/av1_inv_txfm2d.c b/av1/common/av1_inv_txfm2d.c
index 86d16b3..2c01f46 100644
--- a/av1/common/av1_inv_txfm2d.c
+++ b/av1/common/av1_inv_txfm2d.c
@@ -34,6 +34,9 @@
case TXFM_TYPE_IDENTITY8: return av1_iidentity8_c;
case TXFM_TYPE_IDENTITY16: return av1_iidentity16_c;
case TXFM_TYPE_IDENTITY32: return av1_iidentity32_c;
+#if CONFIG_TX64X64
+ case TXFM_TYPE_IDENTITY64: return av1_iidentity64_c;
+#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_TX
default: assert(0); return NULL;
}
@@ -46,14 +49,22 @@
NULL,
#endif
&inv_txfm_1d_col_cfg_dct_4, &inv_txfm_1d_col_cfg_dct_8,
- &inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_32 },
+ &inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_32,
+#if CONFIG_TX64X64
+ &inv_txfm_1d_col_cfg_dct_64
+#endif // CONFIG_TX64X64
+ },
// ADST
{
#if CONFIG_CHROMA_2X2
NULL,
#endif
&inv_txfm_1d_col_cfg_adst_4, &inv_txfm_1d_col_cfg_adst_8,
- &inv_txfm_1d_col_cfg_adst_16, &inv_txfm_1d_col_cfg_adst_32 },
+ &inv_txfm_1d_col_cfg_adst_16, &inv_txfm_1d_col_cfg_adst_32,
+#if CONFIG_TX64X64
+ NULL
+#endif // CONFIG_TX64X64
+ },
#if CONFIG_EXT_TX
// FLIPADST
{
@@ -61,14 +72,22 @@
NULL,
#endif
&inv_txfm_1d_col_cfg_adst_4, &inv_txfm_1d_col_cfg_adst_8,
- &inv_txfm_1d_col_cfg_adst_16, &inv_txfm_1d_col_cfg_adst_32 },
+ &inv_txfm_1d_col_cfg_adst_16, &inv_txfm_1d_col_cfg_adst_32,
+#if CONFIG_TX64X64
+ NULL
+#endif // CONFIG_TX64X64
+ },
// IDENTITY
{
#if CONFIG_CHROMA_2X2
NULL,
#endif
&inv_txfm_1d_cfg_identity_4, &inv_txfm_1d_cfg_identity_8,
- &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32 },
+ &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32,
+#if CONFIG_TX64X64
+ &inv_txfm_1d_cfg_identity_64
+#endif // CONFIG_TX64X64
+ },
#endif // CONFIG_EXT_TX
};
@@ -79,14 +98,22 @@
NULL,
#endif
&inv_txfm_1d_row_cfg_dct_4, &inv_txfm_1d_row_cfg_dct_8,
- &inv_txfm_1d_row_cfg_dct_16, &inv_txfm_1d_row_cfg_dct_32 },
+ &inv_txfm_1d_row_cfg_dct_16, &inv_txfm_1d_row_cfg_dct_32,
+#if CONFIG_TX64X64
+ &inv_txfm_1d_row_cfg_dct_64,
+#endif // CONFIG_TX64X64
+ },
// ADST
{
#if CONFIG_CHROMA_2X2
NULL,
#endif
&inv_txfm_1d_row_cfg_adst_4, &inv_txfm_1d_row_cfg_adst_8,
- &inv_txfm_1d_row_cfg_adst_16, &inv_txfm_1d_row_cfg_adst_32 },
+ &inv_txfm_1d_row_cfg_adst_16, &inv_txfm_1d_row_cfg_adst_32,
+#if CONFIG_TX64X64
+ NULL
+#endif // CONFIG_TX64X64
+ },
#if CONFIG_EXT_TX
// FLIPADST
{
@@ -94,14 +121,22 @@
NULL,
#endif
&inv_txfm_1d_row_cfg_adst_4, &inv_txfm_1d_row_cfg_adst_8,
- &inv_txfm_1d_row_cfg_adst_16, &inv_txfm_1d_row_cfg_adst_32 },
+ &inv_txfm_1d_row_cfg_adst_16, &inv_txfm_1d_row_cfg_adst_32,
+#if CONFIG_TX64X64
+ NULL
+#endif // CONFIG_TX64X64
+ },
// IDENTITY
{
#if CONFIG_CHROMA_2X2
NULL,
#endif
&inv_txfm_1d_cfg_identity_4, &inv_txfm_1d_cfg_identity_8,
- &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32 },
+ &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32,
+#if CONFIG_TX64X64
+ &inv_txfm_1d_cfg_identity_64
+#endif // CONFIG_TX64X64
+ },
#endif // CONFIG_EXT_TX
};
@@ -117,6 +152,7 @@
return cfg;
}
+#if CONFIG_TX64X64
TXFM_2D_FLIP_CFG av1_get_inv_txfm_64x64_cfg(int tx_type) {
TXFM_2D_FLIP_CFG cfg = { 0, 0, NULL, NULL };
switch (tx_type) {
@@ -130,6 +166,33 @@
return cfg;
}
+TXFM_2D_FLIP_CFG av1_get_inv_txfm_32x64_cfg(int tx_type) {
+ TXFM_2D_FLIP_CFG cfg = { 0, 0, NULL, NULL };
+ switch (tx_type) {
+ case DCT_DCT:
+ cfg.col_cfg = &inv_txfm_1d_col_cfg_dct_64;
+ cfg.row_cfg = &inv_txfm_1d_row_cfg_dct_32;
+ set_flip_cfg(tx_type, &cfg);
+ break;
+ default: assert(0);
+ }
+ return cfg;
+}
+
+TXFM_2D_FLIP_CFG av1_get_inv_txfm_64x32_cfg(int tx_type) {
+ TXFM_2D_FLIP_CFG cfg = { 0, 0, NULL, NULL };
+ switch (tx_type) {
+ case DCT_DCT:
+ cfg.col_cfg = &inv_txfm_1d_col_cfg_dct_32;
+ cfg.row_cfg = &inv_txfm_1d_row_cfg_dct_64;
+ set_flip_cfg(tx_type, &cfg);
+ break;
+ default: assert(0);
+ }
+ return cfg;
+}
+#endif // CONFIG_TX64X64
+
void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
const TXFM_2D_FLIP_CFG *cfg, int8_t fwd_shift,
int bd) {
@@ -353,15 +416,30 @@
void av1_inv_txfm2d_add_64x64_c(const int32_t *input, uint16_t *output,
int stride, int tx_type, int bd) {
int txfm_buf[64 * 64 + 64 + 64];
- TXFM_2D_FLIP_CFG cfg = av1_get_inv_txfm_64x64_cfg(tx_type);
- inv_txfm2d_add_c(input, output, stride, &cfg, txfm_buf, -4, bd);
- assert(fwd_shift_sum[TX_64X64] == -4);
+ inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_64X64, bd);
}
void av1_inv_txfm2d_add_64x32_c(const int32_t *input, uint16_t *output,
int stride, int tx_type, int bd) {
+#if CONFIG_TXMG
int txfm_buf[64 * 32 + 64 + 64];
- inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_32X64, bd);
+ int32_t rinput[64 * 32];
+ uint16_t routput[64 * 32];
+ int tx_size = TX_64X32;
+ int rtx_size = av1_rotate_tx_size(tx_size);
+ int rtx_type = av1_rotate_tx_type(tx_type);
+ int w = tx_size_wide[tx_size];
+ int h = tx_size_high[tx_size];
+ int rw = h;
+ int rh = w;
+ transpose_int32(rinput, rw, input, w, w, h);
+ transpose_uint16(routput, rw, output, stride, w, h);
+ inv_txfm2d_add_facade(rinput, routput, rw, txfm_buf, rtx_type, rtx_size, bd);
+ transpose_uint16(output, stride, routput, rw, rw, rh);
+#else
+ int txfm_buf[64 * 32 + 64 + 64];
+ inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_64X32, bd);
+#endif
}
void av1_inv_txfm2d_add_32x64_c(const int32_t *input, uint16_t *output,
diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h
index a61e25d..4c0a2d1 100644
--- a/av1/common/av1_txfm.h
+++ b/av1/common/av1_txfm.h
@@ -134,6 +134,7 @@
TXFM_TYPE_IDENTITY8,
TXFM_TYPE_IDENTITY16,
TXFM_TYPE_IDENTITY32,
+ TXFM_TYPE_IDENTITY64,
} TXFM_TYPE;
typedef struct TXFM_1D_CFG {
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 01214f3..eadd4eb 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -988,9 +988,9 @@
const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
#if CONFIG_CB4X4 && USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
(void)bs;
- if (tx_size_sqr > TX_32X32) return EXT_TX_SET_DCTONLY;
+ if (tx_size_sqr_up > TX_32X32) return EXT_TX_SET_DCTONLY;
#else
- if (tx_size_sqr > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
+ if (tx_size_sqr_up > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
#endif
if (use_reduced_set)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
diff --git a/av1/common/common_data.h b/av1/common/common_data.h
index ec28000..d063122 100644
--- a/av1/common/common_data.h
+++ b/av1/common/common_data.h
@@ -665,9 +665,9 @@
TX_8X16, TX_16X8, TX_16X16,
// 16X32, 32X16, 32X32
TX_16X32, TX_32X16, TX_32X32,
+#if CONFIG_TX64X64
// 32X64, 64X32,
TX_32X32, TX_32X32,
-#if CONFIG_TX64X64
// 64X64
TX_64X64,
#if CONFIG_EXT_PARTITION
@@ -675,6 +675,8 @@
TX_64X64, TX_64X64, TX_64X64,
#endif // CONFIG_EXT_PARTITION
#else
+ // 32X64, 64X32,
+ TX_32X32, TX_32X32,
// 64X64
TX_32X32,
#if CONFIG_EXT_PARTITION
@@ -775,9 +777,9 @@
TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_16X16 - TX_8X8,
// 16X32, 32X16, 32X32
TX_32X32 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
- // 32X64, 64X32,
- TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
#if CONFIG_TX64X64
+ // 32X64, 64X32,
+ TX_64X64 - TX_8X8, TX_64X64 - TX_8X8,
// 64X64
TX_64X64 - TX_8X8,
#if CONFIG_EXT_PARTITION
@@ -785,6 +787,8 @@
TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, TX_64X64 - TX_8X8,
#endif // CONFIG_EXT_PARTITION
#else
+ // 32X64, 64X32,
+ TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
// 64X64
TX_32X32 - TX_8X8,
#if CONFIG_EXT_PARTITION
@@ -818,9 +822,9 @@
TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_16X16 - TX_8X8,
// 16X32, 32X16, 32X32
TX_16X16 - TX_8X8, TX_16X16 - TX_8X8, TX_32X32 - TX_8X8,
- // 32X64, 64X32,
- TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
#if CONFIG_TX64X64
+ // 32X64, 64X32,
+ TX_64X64 - TX_8X8, TX_64X64 - TX_8X8,
// 64X64
TX_64X64 - TX_8X8,
#if CONFIG_EXT_PARTITION
@@ -828,6 +832,8 @@
TX_64X64 - TX_8X8, TX_64X64 - TX_8X8, TX_64X64 - TX_8X8,
#endif // CONFIG_EXT_PARTITION
#else
+ // 32X64, 64X32,
+ TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
// 64X64
TX_32X32 - TX_8X8,
#if CONFIG_EXT_PARTITION
diff --git a/av1/common/idct.c b/av1/common/idct.c
index ca1c361..5cde3b7 100644
--- a/av1/common/idct.c
+++ b/av1/common/idct.c
@@ -1566,7 +1566,7 @@
for (i = 0; i < n; ++i) {
IHT_64x32[tx_type].rows(input, outtmp);
for (j = 0; j < n2; ++j)
- tmp[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2);
+ tmp[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * InvSqrt2);
input += n2;
}
@@ -1628,7 +1628,7 @@
for (i = 0; i < n2; ++i) {
IHT_32x64[tx_type].rows(input, outtmp);
for (j = 0; j < n; ++j)
- tmp[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * Sqrt2);
+ tmp[j][i] = (tran_low_t)dct_const_round_shift(outtmp[j] * InvSqrt2);
input += n;
}
@@ -2107,6 +2107,7 @@
static void inv_txfm_add_64x64(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
const TX_TYPE tx_type = txfm_param->tx_type;
+ assert(tx_type == DCT_DCT);
switch (tx_type) {
#if !CONFIG_DAALA_DCT64
case DCT_DCT: idct64x64_add(input, dest, stride, txfm_param); break;
diff --git a/av1/common/scan.c b/av1/common/scan.c
index 146e748..a5a1a9f 100644
--- a/av1/common/scan.c
+++ b/av1/common/scan.c
@@ -4572,7 +4572,7 @@
1726, 1789, 1789, 1852, 1852, 1915, 1915, 1978, 1978, 2041, 1727, 1790, 1790,
1853, 1853, 1916, 1916, 1979, 1979, 2042, 1791, 1854, 1854, 1917, 1917, 1980,
1980, 2043, 1855, 1918, 1918, 1981, 1981, 2044, 1919, 1982, 1982, 2045, 1983,
- 2046,
+ 2046, 0, 0
};
DECLARE_ALIGNED(16, static const int16_t,
