Reduce shift in txfm8x8 Change-Id: I67543d365cbef3c3e113f01660ae8cb744cc556d
diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h index c0d06ba..e8c82fd 100644 --- a/vp10/common/vp10_fwd_txfm2d_cfg.h +++ b/vp10/common/vp10_fwd_txfm2d_cfg.h
@@ -32,11 +32,11 @@ TXFM_TYPE_DCT4}; // .txfm_type_row // ---------------- config fwd_dct_dct_8 ---------------- -static const int8_t fwd_shift_dct_dct_8[3] = {3, -1, -1}; -static const int8_t fwd_stage_range_col_dct_dct_8[6] = {16, 17, 18, 19, 19, 19}; -static const int8_t fwd_stage_range_row_dct_dct_8[6] = {18, 19, 20, 20, 20, 20}; +static const int8_t fwd_shift_dct_dct_8[3] = {2, -1, 0}; +static const int8_t fwd_stage_range_col_dct_dct_8[6] = {15, 16, 17, 18, 18, 18}; +static const int8_t fwd_stage_range_row_dct_dct_8[6] = {17, 18, 19, 19, 19, 19}; static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {12, 12, 12, 12, 12, 12}; +static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = { 8, // .txfm_size @@ -144,14 +144,14 @@ TXFM_TYPE_ADST4}; // .txfm_type_row // ---------------- config fwd_dct_adst_8 ---------------- -static const int8_t fwd_shift_dct_adst_8[3] = {3, -1, -1}; -static const int8_t fwd_stage_range_col_dct_adst_8[6] = {16, 17, 18, - 19, 19, 19}; -static const int8_t fwd_stage_range_row_dct_adst_8[8] = {18, 18, 18, 19, - 19, 20, 20, 20}; +static const int8_t fwd_shift_dct_adst_8[3] = {2, -1, 0}; +static const int8_t fwd_stage_range_col_dct_adst_8[6] = {15, 16, 17, + 18, 18, 18}; +static const int8_t fwd_stage_range_row_dct_adst_8[8] = {17, 17, 17, 18, + 18, 19, 19, 19}; static const int8_t fwd_cos_bit_col_dct_adst_8[6] = {13, 13, 13, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_dct_adst_8[8] = {12, 12, 12, 12, - 12, 12, 12, 12}; +static const int8_t fwd_cos_bit_row_dct_adst_8[8] = {13, 13, 13, 13, + 13, 13, 13, 13}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_adst_8 = { 8, // .txfm_size @@ -236,15 +236,15 @@ TXFM_TYPE_ADST4}; // .txfm_type_row // ---------------- config fwd_adst_adst_8 ---------------- -static const int8_t fwd_shift_adst_adst_8[3] = {3, -1, -1}; -static const int8_t fwd_stage_range_col_adst_adst_8[8] = {16, 16, 17, 18, +static const int8_t fwd_shift_adst_adst_8[3] = {2, -1, 0}; +static const int8_t fwd_stage_range_col_adst_adst_8[8] = {15, 15, 16, 17, + 17, 18, 18, 18}; +static const int8_t fwd_stage_range_row_adst_adst_8[8] = {17, 17, 17, 18, 18, 19, 19, 19}; -static const int8_t fwd_stage_range_row_adst_adst_8[8] = {18, 18, 18, 19, - 19, 20, 20, 20}; static const int8_t fwd_cos_bit_col_adst_adst_8[8] = {13, 13, 13, 13, 13, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_adst_adst_8[8] = {12, 12, 12, 12, - 12, 12, 12, 12}; +static const int8_t fwd_cos_bit_row_adst_adst_8[8] = {13, 13, 13, 13, + 13, 13, 13, 13}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_adst_8 = { 8, // .txfm_size @@ -329,14 +329,14 @@ TXFM_TYPE_DCT4}; // .txfm_type_row // ---------------- config fwd_adst_dct_8 ---------------- -static const int8_t fwd_shift_adst_dct_8[3] = {3, -1, -1}; -static const int8_t fwd_stage_range_col_adst_dct_8[8] = {16, 16, 17, 18, - 18, 19, 19, 19}; -static const int8_t fwd_stage_range_row_adst_dct_8[6] = {18, 19, 20, - 20, 20, 20}; +static const int8_t fwd_shift_adst_dct_8[3] = {2, -1, 0}; +static const int8_t fwd_stage_range_col_adst_dct_8[8] = {15, 15, 16, 17, + 17, 18, 18, 18}; +static const int8_t fwd_stage_range_row_adst_dct_8[6] = {17, 18, 19, + 19, 19, 19}; static const int8_t fwd_cos_bit_col_adst_dct_8[8] = {13, 13, 13, 13, 13, 13, 13, 13}; -static const int8_t fwd_cos_bit_row_adst_dct_8[6] = {12, 12, 12, 12, 12, 12}; +static const int8_t fwd_cos_bit_row_adst_dct_8[6] = {13, 13, 13, 13, 13, 13}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_adst_dct_8 = { 8, // .txfm_size
diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h index 74c8c41..9199068 100644 --- a/vp10/common/vp10_inv_txfm2d_cfg.h +++ b/vp10/common/vp10_inv_txfm2d_cfg.h
@@ -32,10 +32,10 @@ TXFM_TYPE_DCT4}; // .txfm_type_row // ---------------- config inv_dct_dct_8 ---------------- -static const int8_t inv_shift_dct_dct_8[2] = {1, -6}; -static const int8_t inv_stage_range_col_dct_dct_8[6] = {20, 20, 20, 20, 19, 19}; +static const int8_t inv_shift_dct_dct_8[2] = {0, -5}; +static const int8_t inv_stage_range_col_dct_dct_8[6] = {19, 19, 19, 19, 18, 18}; static const int8_t inv_stage_range_row_dct_dct_8[6] = {19, 19, 19, 19, 19, 19}; -static const int8_t inv_cos_bit_col_dct_dct_8[6] = {12, 12, 12, 12, 12, 12}; +static const int8_t inv_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13}; static const int8_t inv_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13}; static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = { @@ -144,12 +144,12 @@ TXFM_TYPE_ADST4}; // .txfm_type_row // ---------------- config inv_dct_adst_8 ---------------- -static const int8_t inv_shift_dct_adst_8[2] = {1, -6}; -static const int8_t inv_stage_range_col_dct_adst_8[6] = {20, 20, 20, - 20, 19, 19}; +static const int8_t inv_shift_dct_adst_8[2] = {0, -5}; +static const int8_t inv_stage_range_col_dct_adst_8[6] = {19, 19, 19, + 19, 18, 18}; static const int8_t inv_stage_range_row_dct_adst_8[8] = {19, 19, 19, 19, 19, 19, 19, 19}; -static const int8_t inv_cos_bit_col_dct_adst_8[6] = {12, 12, 12, 12, 12, 12}; +static const int8_t inv_cos_bit_col_dct_adst_8[6] = {13, 13, 13, 13, 13, 13}; static const int8_t inv_cos_bit_row_dct_adst_8[8] = {13, 13, 13, 13, 13, 13, 13, 13}; @@ -237,13 +237,13 @@ TXFM_TYPE_ADST4}; // .txfm_type_row // ---------------- config inv_adst_adst_8 ---------------- -static const int8_t inv_shift_adst_adst_8[2] = {1, -6}; -static const int8_t inv_stage_range_col_adst_adst_8[8] = {20, 20, 20, 20, - 20, 20, 19, 19}; +static const int8_t inv_shift_adst_adst_8[2] = {0, -5}; +static const int8_t inv_stage_range_col_adst_adst_8[8] = {19, 19, 19, 19, + 19, 19, 18, 18}; static const int8_t inv_stage_range_row_adst_adst_8[8] = {19, 19, 19, 19, 19, 19, 19, 19}; -static const int8_t inv_cos_bit_col_adst_adst_8[8] = {12, 12, 12, 12, - 12, 12, 12, 12}; +static const int8_t inv_cos_bit_col_adst_adst_8[8] = {13, 13, 13, 13, + 13, 13, 13, 13}; static const int8_t inv_cos_bit_row_adst_adst_8[8] = {13, 13, 13, 13, 13, 13, 13, 13}; @@ -330,13 +330,13 @@ TXFM_TYPE_DCT4}; // .txfm_type_row // ---------------- config inv_adst_dct_8 ---------------- -static const int8_t inv_shift_adst_dct_8[2] = {1, -6}; -static const int8_t inv_stage_range_col_adst_dct_8[8] = {20, 20, 20, 20, - 20, 20, 19, 19}; +static const int8_t inv_shift_adst_dct_8[2] = {0, -5}; +static const int8_t inv_stage_range_col_adst_dct_8[8] = {19, 19, 19, 19, + 19, 19, 18, 18}; static const int8_t inv_stage_range_row_adst_dct_8[6] = {19, 19, 19, 19, 19, 19}; -static const int8_t inv_cos_bit_col_adst_dct_8[8] = {12, 12, 12, 12, - 12, 12, 12, 12}; +static const int8_t inv_cos_bit_col_adst_dct_8[8] = {13, 13, 13, 13, + 13, 13, 13, 13}; static const int8_t inv_cos_bit_row_adst_dct_8[6] = {13, 13, 13, 13, 13, 13}; static const TXFM_2D_CFG inv_txfm_2d_cfg_adst_dct_8 = {