Reduce shift in txfm8x8
Change-Id: I67543d365cbef3c3e113f01660ae8cb744cc556d
diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h
index c0d06ba..e8c82fd 100644
--- a/vp10/common/vp10_fwd_txfm2d_cfg.h
+++ b/vp10/common/vp10_fwd_txfm2d_cfg.h
@@ -32,11 +32,11 @@
TXFM_TYPE_DCT4}; // .txfm_type_row
// ---------------- config fwd_dct_dct_8 ----------------
-static const int8_t fwd_shift_dct_dct_8[3] = {3, -1, -1};
-static const int8_t fwd_stage_range_col_dct_dct_8[6] = {16, 17, 18, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_8[6] = {18, 19, 20, 20, 20, 20};
+static const int8_t fwd_shift_dct_dct_8[3] = {2, -1, 0};
+static const int8_t fwd_stage_range_col_dct_dct_8[6] = {15, 16, 17, 18, 18, 18};
+static const int8_t fwd_stage_range_row_dct_dct_8[6] = {17, 18, 19, 19, 19, 19};
static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {12, 12, 12, 12, 12, 12};
+static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = {
8, // .txfm_size
@@ -144,14 +144,14 @@
TXFM_TYPE_ADST4}; // .txfm_type_row
// ---------------- config fwd_dct_adst_8 ----------------
-static const int8_t fwd_shift_dct_adst_8[3] = {3, -1, -1};
-static const int8_t fwd_stage_range_col_dct_adst_8[6] = {16, 17, 18,
- 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_adst_8[8] = {18, 18, 18, 19,
- 19, 20, 20, 20};
+static const int8_t fwd_shift_dct_adst_8[3] = {2, -1, 0};
+static const int8_t fwd_stage_range_col_dct_adst_8[6] = {15, 16, 17,
+ 18, 18, 18};
+static const int8_t fwd_stage_range_row_dct_adst_8[8] = {17, 17, 17, 18,
+ 18, 19, 19, 19};
static const int8_t fwd_cos_bit_col_dct_adst_8[6] = {13, 13, 13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_adst_8[8] = {12, 12, 12, 12,
- 12, 12, 12, 12};
+static const int8_t fwd_cos_bit_row_dct_adst_8[8] = {13, 13, 13, 13,
+ 13, 13, 13, 13};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_adst_8 = {
8, // .txfm_size
@@ -236,15 +236,15 @@
TXFM_TYPE_ADST4}; // .txfm_type_row
// ---------------- config fwd_adst_adst_8 ----------------
-static const int8_t fwd_shift_adst_adst_8[3] = {3, -1, -1};
-static const int8_t fwd_stage_range_col_adst_adst_8[8] = {16, 16, 17, 18,
+static const int8_t fwd_shift_adst_adst_8[3] = {2, -1, 0};
+static const int8_t fwd_stage_range_col_adst_adst_8[8] = {15, 15, 16, 17,
+ 17, 18, 18, 18};
+static const int8_t fwd_stage_range_row_adst_adst_8[8] = {17, 17, 17, 18,
18, 19, 19, 19};
-static const int8_t fwd_stage_range_row_adst_adst_8[8] = {18, 18, 18, 19,
- 19, 20, 20, 20};
static const int8_t fwd_cos_bit_col_adst_adst_8[8] = {13, 13, 13, 13,
13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_adst_adst_8[8] = {12, 12, 12, 12,
- 12, 12, 12, 12};
+static const int8_t fwd_cos_bit_row_adst_adst_8[8] = {13, 13, 13, 13,
+ 13, 13, 13, 13};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_adst_8 = {
8, // .txfm_size
@@ -329,14 +329,14 @@
TXFM_TYPE_DCT4}; // .txfm_type_row
// ---------------- config fwd_adst_dct_8 ----------------
-static const int8_t fwd_shift_adst_dct_8[3] = {3, -1, -1};
-static const int8_t fwd_stage_range_col_adst_dct_8[8] = {16, 16, 17, 18,
- 18, 19, 19, 19};
-static const int8_t fwd_stage_range_row_adst_dct_8[6] = {18, 19, 20,
- 20, 20, 20};
+static const int8_t fwd_shift_adst_dct_8[3] = {2, -1, 0};
+static const int8_t fwd_stage_range_col_adst_dct_8[8] = {15, 15, 16, 17,
+ 17, 18, 18, 18};
+static const int8_t fwd_stage_range_row_adst_dct_8[6] = {17, 18, 19,
+ 19, 19, 19};
static const int8_t fwd_cos_bit_col_adst_dct_8[8] = {13, 13, 13, 13,
13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_adst_dct_8[6] = {12, 12, 12, 12, 12, 12};
+static const int8_t fwd_cos_bit_row_adst_dct_8[6] = {13, 13, 13, 13, 13, 13};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_dct_8 = {
8, // .txfm_size
diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h
index 74c8c41..9199068 100644
--- a/vp10/common/vp10_inv_txfm2d_cfg.h
+++ b/vp10/common/vp10_inv_txfm2d_cfg.h
@@ -32,10 +32,10 @@
TXFM_TYPE_DCT4}; // .txfm_type_row
// ---------------- config inv_dct_dct_8 ----------------
-static const int8_t inv_shift_dct_dct_8[2] = {1, -6};
-static const int8_t inv_stage_range_col_dct_dct_8[6] = {20, 20, 20, 20, 19, 19};
+static const int8_t inv_shift_dct_dct_8[2] = {0, -5};
+static const int8_t inv_stage_range_col_dct_dct_8[6] = {19, 19, 19, 19, 18, 18};
static const int8_t inv_stage_range_row_dct_dct_8[6] = {19, 19, 19, 19, 19, 19};
-static const int8_t inv_cos_bit_col_dct_dct_8[6] = {12, 12, 12, 12, 12, 12};
+static const int8_t inv_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
static const int8_t inv_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = {
@@ -144,12 +144,12 @@
TXFM_TYPE_ADST4}; // .txfm_type_row
// ---------------- config inv_dct_adst_8 ----------------
-static const int8_t inv_shift_dct_adst_8[2] = {1, -6};
-static const int8_t inv_stage_range_col_dct_adst_8[6] = {20, 20, 20,
- 20, 19, 19};
+static const int8_t inv_shift_dct_adst_8[2] = {0, -5};
+static const int8_t inv_stage_range_col_dct_adst_8[6] = {19, 19, 19,
+ 19, 18, 18};
static const int8_t inv_stage_range_row_dct_adst_8[8] = {19, 19, 19, 19,
19, 19, 19, 19};
-static const int8_t inv_cos_bit_col_dct_adst_8[6] = {12, 12, 12, 12, 12, 12};
+static const int8_t inv_cos_bit_col_dct_adst_8[6] = {13, 13, 13, 13, 13, 13};
static const int8_t inv_cos_bit_row_dct_adst_8[8] = {13, 13, 13, 13,
13, 13, 13, 13};
@@ -237,13 +237,13 @@
TXFM_TYPE_ADST4}; // .txfm_type_row
// ---------------- config inv_adst_adst_8 ----------------
-static const int8_t inv_shift_adst_adst_8[2] = {1, -6};
-static const int8_t inv_stage_range_col_adst_adst_8[8] = {20, 20, 20, 20,
- 20, 20, 19, 19};
+static const int8_t inv_shift_adst_adst_8[2] = {0, -5};
+static const int8_t inv_stage_range_col_adst_adst_8[8] = {19, 19, 19, 19,
+ 19, 19, 18, 18};
static const int8_t inv_stage_range_row_adst_adst_8[8] = {19, 19, 19, 19,
19, 19, 19, 19};
-static const int8_t inv_cos_bit_col_adst_adst_8[8] = {12, 12, 12, 12,
- 12, 12, 12, 12};
+static const int8_t inv_cos_bit_col_adst_adst_8[8] = {13, 13, 13, 13,
+ 13, 13, 13, 13};
static const int8_t inv_cos_bit_row_adst_adst_8[8] = {13, 13, 13, 13,
13, 13, 13, 13};
@@ -330,13 +330,13 @@
TXFM_TYPE_DCT4}; // .txfm_type_row
// ---------------- config inv_adst_dct_8 ----------------
-static const int8_t inv_shift_adst_dct_8[2] = {1, -6};
-static const int8_t inv_stage_range_col_adst_dct_8[8] = {20, 20, 20, 20,
- 20, 20, 19, 19};
+static const int8_t inv_shift_adst_dct_8[2] = {0, -5};
+static const int8_t inv_stage_range_col_adst_dct_8[8] = {19, 19, 19, 19,
+ 19, 19, 18, 18};
static const int8_t inv_stage_range_row_adst_dct_8[6] = {19, 19, 19,
19, 19, 19};
-static const int8_t inv_cos_bit_col_adst_dct_8[8] = {12, 12, 12, 12,
- 12, 12, 12, 12};
+static const int8_t inv_cos_bit_col_adst_dct_8[8] = {13, 13, 13, 13,
+ 13, 13, 13, 13};
static const int8_t inv_cos_bit_row_adst_dct_8[6] = {13, 13, 13, 13, 13, 13};
static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_dct_8 = {
