Tune fwd txfm's config
Maximize cos_bit's precision
Change-Id: Iad5d3915823f5c1c25a0caa3bd012d60caa2d521
diff --git a/av1/encoder/av1_fwd_txfm1d_cfg.h b/av1/encoder/av1_fwd_txfm1d_cfg.h
index 21e4b45..84a4340 100644
--- a/av1/encoder/av1_fwd_txfm1d_cfg.h
+++ b/av1/encoder/av1_fwd_txfm1d_cfg.h
@@ -109,15 +109,15 @@
// cos bit
static const int8_t fwd_cos_bit_col_dct_64[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
-static const int8_t fwd_cos_bit_row_dct_64[12] = { 13, 13, 12, 11, 10, 10,
+static const int8_t fwd_cos_bit_row_dct_64[12] = { 10, 10, 10, 10, 10, 10,
10, 10, 10, 10, 10, 10 };
// ---------------- 4x8 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_4x8[4] = { 3, 4, 4, 4 };
static const int8_t fwd_stage_range_row_adst_4x8[6] = { 3, 3, 3, 4, 4, 4 };
static const int8_t fwd_stage_range_row_idx_4x8[1] = { 4 };
-static const int8_t fwd_cos_bit_row_dct_4x8[6] = { 13, 12, 12, 12 };
-static const int8_t fwd_cos_bit_row_adst_4x8[6] = { 13, 13, 12, 12, 12, 12 };
+static const int8_t fwd_cos_bit_row_dct_4x8[6] = { 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_4x8[6] = { 13, 13, 13, 13, 13, 13 };
// ---------------- 8x4 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_8x4[6] = { 2, 3, 4, 4, 4, 4 };
@@ -125,9 +125,9 @@
2, 2, 2, 3, 3, 4, 4, 4
};
static const int8_t fwd_stage_range_row_idx_8x4[1] = { 3 };
-static const int8_t fwd_cos_bit_row_dct_8x4[6] = { 13, 13, 12, 12, 12, 12 };
+static const int8_t fwd_cos_bit_row_dct_8x4[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_8x4[8] = { 13, 13, 13, 13,
- 12, 12, 12, 12 };
+ 13, 13, 13, 13 };
// ---------------- 8x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_8x16[6] = { 4, 5, 6, 6, 6, 6 };
@@ -135,9 +135,9 @@
4, 4, 4, 5, 5, 6, 6, 6
};
static const int8_t fwd_stage_range_row_idx_8x16[1] = { 5 };
-static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 12, 12, 11, 11, 11, 11 };
-static const int8_t fwd_cos_bit_row_adst_8x16[8] = { 12, 12, 12, 12,
- 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 13, 13, 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_8x16[8] = { 13, 13, 13, 13,
+ 13, 13, 13, 13 };
// ---------------- 16x8 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x8[8] = {
@@ -146,10 +146,10 @@
static const int8_t fwd_stage_range_row_adst_16x8[10] = { 3, 3, 3, 4, 4,
5, 5, 6, 6, 6 };
static const int8_t fwd_stage_range_row_idx_16x8[1] = { 5 };
-static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 12, 12, 12, 11,
- 11, 11, 11, 11 };
-static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 12, 12, 12, 12, 12,
- 12, 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 13, 13, 13, 13,
+ 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13 };
// ---------------- 16x32 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x32[8] = {
@@ -158,10 +158,10 @@
static const int8_t fwd_stage_range_row_adst_16x32[10] = { 5, 5, 5, 6, 6,
7, 7, 8, 8, 8 };
static const int8_t fwd_stage_range_row_idx_16x32[1] = { 7 };
-static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 12, 12, 12, 11,
- 11, 11, 11, 11 };
-static const int8_t fwd_cos_bit_row_adst_16x32[10] = { 12, 12, 12, 12, 12,
- 12, 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 13, 13, 13, 13,
+ 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_16x32[10] = { 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13 };
// ---------------- 32x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_32x16[10] = { 4, 5, 6, 7, 8,
@@ -169,32 +169,32 @@
static const int8_t fwd_stage_range_row_adst_32x16[12] = { 4, 4, 4, 5, 5, 6,
6, 7, 7, 8, 8, 8 };
static const int8_t fwd_stage_range_row_idx_32x16[1] = { 6 };
-static const int8_t fwd_cos_bit_row_dct_32x16[10] = { 12, 12, 12, 12, 11,
- 11, 11, 11, 11, 11 };
-static const int8_t fwd_cos_bit_row_adst_32x16[12] = { 12, 12, 12, 12, 12, 12,
- 12, 12, 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_32x16[10] = { 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_32x16[12] = { 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13 };
// ---------------- 32x64 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_32x64[10] = { 6, 7, 8, 9, 10,
10, 10, 10, 10, 10 };
static const int8_t fwd_stage_range_row_idx_32x64[1] = { 8 };
-static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 13, 12, 12, 11, 10,
- 10, 10, 10, 10, 10 };
+static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11 };
// ---------------- 64x32 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_64x32[12] = {
5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 10, 10
};
static const int8_t fwd_stage_range_row_idx_64x32[1] = { 8 };
-static const int8_t fwd_cos_bit_row_dct_64x32[12] = { 13, 13, 12, 11, 10, 10,
- 10, 10, 10, 10, 10, 10 };
+static const int8_t fwd_cos_bit_row_dct_64x32[12] = { 11, 11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11, 11 };
// ---------------- 4x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_4x16[4] = { 4, 5, 5, 5 };
static const int8_t fwd_stage_range_row_adst_4x16[6] = { 4, 4, 4, 5, 5, 5 };
static const int8_t fwd_stage_range_row_idx_4x16[1] = { 5 };
-static const int8_t fwd_cos_bit_row_dct_4x16[6] = { 12, 12, 12, 12 };
-static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 12, 12, 12, 12, 12, 12 };
+static const int8_t fwd_cos_bit_row_dct_4x16[6] = { 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 13, 13, 13, 13, 13, 13 };
// ---------------- 16x4 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x4[8] = {
@@ -203,10 +203,10 @@
static const int8_t fwd_stage_range_row_adst_16x4[10] = { 2, 2, 2, 3, 3,
4, 4, 5, 5, 5 };
static const int8_t fwd_stage_range_row_idx_16x4[1] = { 4 };
-static const int8_t fwd_cos_bit_row_dct_16x4[8] = { 12, 12, 12, 12,
- 12, 12, 12, 12 };
-static const int8_t fwd_cos_bit_row_adst_16x4[10] = { 12, 12, 12, 12, 12,
- 12, 12, 12, 12, 12 };
+static const int8_t fwd_cos_bit_row_dct_16x4[8] = { 13, 13, 13, 13,
+ 13, 13, 13, 13 };
+static const int8_t fwd_cos_bit_row_adst_16x4[10] = { 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13 };
// ---------------- 8x32 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_8x32[6] = { 5, 6, 7, 7, 7, 7 };
@@ -214,9 +214,9 @@
5, 5, 5, 6, 6, 7, 7, 7
};
static const int8_t fwd_stage_range_row_idx_8x32[1] = { 6 };
-static const int8_t fwd_cos_bit_row_dct_8x32[6] = { 12, 12, 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_8x32[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_8x32[8] = { 12, 12, 12, 12,
- 11, 11, 11, 11 };
+ 12, 12, 12, 12 };
// ---------------- 32x8 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_32x8[10] = { 3, 4, 5, 6, 7,
@@ -224,25 +224,25 @@
static const int8_t fwd_stage_range_row_adst_32x8[12] = { 3, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 7 };
static const int8_t fwd_stage_range_row_idx_32x8[1] = { 5 };
-static const int8_t fwd_cos_bit_row_dct_32x8[10] = { 12, 12, 12, 12, 11,
- 11, 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_32x8[10] = { 12, 12, 12, 12, 12,
+ 12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_32x8[12] = { 12, 12, 12, 12, 12, 12,
- 12, 11, 11, 11, 11, 11 };
+ 12, 12, 12, 12, 12, 12 };
// ---------------- 16x64 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x64[8] = {
6, 7, 8, 9, 9, 9, 9, 9
};
static const int8_t fwd_stage_range_row_idx_16x64[1] = { 8 };
-static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 11, 10, 10,
- 10, 10, 10, 10 };
+static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 12, 12, 12,
+ 12, 12, 12, 12 };
// ---------------- 64x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_64x16[12] = { 4, 5, 6, 7, 8, 9,
9, 9, 9, 9, 9, 9 };
static const int8_t fwd_stage_range_row_idx_64x16[1] = { 7 };
-static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 13, 13, 12, 11, 11, 11,
- 11, 11, 11, 11, 11, 11 };
+static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 12, 12, 12, 12, 12, 12,
+ 12, 12, 12, 12, 12, 12 };
//
// ---------------- row config fwd_dct_4 ----------------
diff --git a/av1/encoder/av1_fwd_txfm2d.c b/av1/encoder/av1_fwd_txfm2d.c
index 045d67e..fee856e 100644
--- a/av1/encoder/av1_fwd_txfm2d.c
+++ b/av1/encoder/av1_fwd_txfm2d.c
@@ -631,7 +631,7 @@
static const int8_t fwd_shift_32x16[3] = { 2, -4, 0 };
#if CONFIG_TX64X64
static const int8_t fwd_shift_32x64[3] = { 0, -2, -2 };
-static const int8_t fwd_shift_64x32[3] = { 0, -2, -2 };
+static const int8_t fwd_shift_64x32[3] = { 2, -4, -2 };
#endif
static const int8_t fwd_shift_4x16[3] = { 2, -1, 0 };
static const int8_t fwd_shift_16x4[3] = { 2, -1, 0 };
@@ -639,7 +639,7 @@
static const int8_t fwd_shift_32x8[3] = { 2, -2, 0 };
#if CONFIG_TX64X64
static const int8_t fwd_shift_16x64[3] = { 0, -2, 0 };
-static const int8_t fwd_shift_64x16[3] = { 0, -2, 0 };
+static const int8_t fwd_shift_64x16[3] = { 2, -4, 0 };
#endif // CONFIG_TX64X64
const int8_t *fwd_txfm_shift_ls[TX_SIZES_ALL] = {
diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc
index 2ab629f..420c7e9 100644
--- a/test/av1_fwd_txfm2d_test.cc
+++ b/test/av1_fwd_txfm2d_test.cc
@@ -139,33 +139,33 @@
for (int t = 0; t < TX_TYPES; ++t) {
const TX_TYPE tx_type = static_cast<TX_TYPE>(t);
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X4, 2, 0.5));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.6));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.5));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 7));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.5));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.2));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 6.1));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 70, 7));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 64, 3.4));
}
#endif // CONFIG_TX64X64
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.58));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.6, 0.63));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 15, 1.5));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 15, 1.5));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 57, 7));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 7));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.50));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.64));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 8, 0.8));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 8, 1.1));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 29, 3.9));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 5.9));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 5, 0.7));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5.5, 0.9));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 20, 2.2));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 15, 1.8));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 3, 0.6));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5, 0.9));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 21, 1.2));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 13, 1.7));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 3.6));
- param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 50, 6.0));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 2.9));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 4.9));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 2.0));
+ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 36, 4.7));
}
#endif // CONFIG_TX64X64
}
diff --git a/test/av1_inv_txfm2d_test.cc b/test/av1_inv_txfm2d_test.cc
index 6260424..6a50723 100644
--- a/test/av1_inv_txfm2d_test.cc
+++ b/test/av1_inv_txfm2d_test.cc
@@ -155,7 +155,7 @@
#endif // CONFIG_TX64X64
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X8, 2, 0.016));
- param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.025));
+ param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.045));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X16, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X8, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X32, 3, 0.4));
