Change dct32x32's range Bitdepth 10/12: Fit coefficient range into 32 bits Fit codfficient * const range into 32 bits Bitdepth 8: Fit coefficient range into 16 bits Fit codfficient * constant range into 32 bits Change-Id: I50b5a3132e8a9f5155c971ab0f6eb52876d2b5ca
diff --git a/vp10/common/idct.c b/vp10/common/idct.c index b06a5e9..0eec3b4 100644 --- a/vp10/common/idct.c +++ b/vp10/common/idct.c
@@ -24,11 +24,7 @@ (void) tx_type; #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - if (xd->bd == BITDEPTH_10) { - return 0; - } else { - return tx_size == TX_32X32; - } + return tx_size == TX_32X32; } #else (void)xd;
diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h index 2f60baa..10050b7 100644 --- a/vp10/common/vp10_fwd_txfm2d_cfg.h +++ b/vp10/common/vp10_fwd_txfm2d_cfg.h
@@ -76,21 +76,21 @@ TXFM_TYPE_DCT16}; // .txfm_type_row // ---------------- config fwd_dct_dct_32 ---------------- -static const int8_t fwd_shift_dct_dct_32[3] = {2, -3, 0}; +static const int8_t fwd_shift_dct_dct_32[3] = {2, -4, 0}; static const int8_t fwd_stage_range_col_dct_dct_32[10] = {15, 16, 17, 18, 19, 20, 20, 20, 20, 20}; -static const int8_t fwd_stage_range_row_dct_dct_32[10] = {17, 18, 19, 20, 21, - 21, 21, 21, 21, 21}; +static const int8_t fwd_stage_range_row_dct_dct_32[10] = {16, 17, 18, 19, 20, + 20, 20, 20, 20, 20}; static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13, 12, 12, 12, 12, 12}; -static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 12, 11, - 11, 11, 11, 11, 11}; +static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 13, 12, + 12, 12, 12, 12, 12}; static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = { 32, // .txfm_size 10, // .stage_num_col 10, // .stage_num_row - // 0, // .log_scale + // 1, // .log_scale fwd_shift_dct_dct_32, // .shift fwd_stage_range_col_dct_dct_32, // .stage_range_col fwd_stage_range_row_dct_dct_32, // .stage_range_row
diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h index 897e21c..f1bb2c7 100644 --- a/vp10/common/vp10_inv_txfm2d_cfg.h +++ b/vp10/common/vp10_inv_txfm2d_cfg.h
@@ -76,21 +76,21 @@ TXFM_TYPE_DCT16}; // .txfm_type_row // ---------------- config inv_dct_dct_32 ---------------- -static const int8_t inv_shift_dct_dct_32[2] = {-1, -6}; -static const int8_t inv_stage_range_col_dct_dct_32[10] = {20, 20, 20, 20, 20, - 20, 20, 20, 19, 19}; -static const int8_t inv_stage_range_row_dct_dct_32[10] = {21, 21, 21, 21, 21, - 21, 21, 21, 21, 21}; -static const int8_t inv_cos_bit_col_dct_dct_32[10] = {12, 12, 12, 12, 12, - 12, 12, 12, 12, 13}; -static const int8_t inv_cos_bit_row_dct_dct_32[10] = {11, 11, 11, 11, 11, - 11, 11, 11, 11, 11}; +static const int8_t inv_shift_dct_dct_32[2] = {-1, -5}; +static const int8_t inv_stage_range_col_dct_dct_32[10] = {19, 19, 19, 19, 19, + 19, 19, 19, 18, 18}; +static const int8_t inv_stage_range_row_dct_dct_32[10] = {20, 20, 20, 20, 20, + 20, 20, 20, 20, 20}; +static const int8_t inv_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13, + 13, 13, 13, 13, 13}; +static const int8_t inv_cos_bit_row_dct_dct_32[10] = {12, 12, 12, 12, 12, + 12, 12, 12, 12, 12}; static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = { 32, // .txfm_size 10, // .stage_num_col 10, // .stage_num_row - // 0, // .log_scale + // 1, // .log_scale inv_shift_dct_dct_32, // .shift inv_stage_range_col_dct_dct_32, // .stage_range_col inv_stage_range_row_dct_dct_32, // .stage_range_row