Fix 2x2 high bit-depth transform setups This commit fixes the 2x2 transform system setups for high bit- depth setting. It enables the cb4x4 mode to support high bit-depth process. The coding performance is improved over high bit-depth + ref-mv: lowres 2.5% midres 1.2% Change-Id: I351f9d72bdc7e15b2bd00e94286b98966a295e6d
diff --git a/av1/common/av1_fwd_txfm2d.c b/av1/common/av1_fwd_txfm2d.c index 2bba193..41f3e71 100644 --- a/av1/common/av1_fwd_txfm2d.c +++ b/av1/common/av1_fwd_txfm2d.c
@@ -138,16 +138,32 @@ { &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, }; -#else // CONFIG_EXT_TX +#else // CONFIG_EXT_TX static const TXFM_2D_CFG *fwd_txfm_cfg_ls[TX_TYPES][TX_SIZES] = { - { &fwd_txfm_2d_cfg_dct_dct_4, &fwd_txfm_2d_cfg_dct_dct_8, - &fwd_txfm_2d_cfg_dct_dct_16, &fwd_txfm_2d_cfg_dct_dct_32 }, - { &fwd_txfm_2d_cfg_adst_dct_4, &fwd_txfm_2d_cfg_adst_dct_8, - &fwd_txfm_2d_cfg_adst_dct_16, &fwd_txfm_2d_cfg_adst_dct_32 }, - { &fwd_txfm_2d_cfg_dct_adst_4, &fwd_txfm_2d_cfg_dct_adst_8, - &fwd_txfm_2d_cfg_dct_adst_16, &fwd_txfm_2d_cfg_dct_adst_32 }, - { &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, - &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_dct_4, &fwd_txfm_2d_cfg_dct_dct_8, + &fwd_txfm_2d_cfg_dct_dct_16, &fwd_txfm_2d_cfg_dct_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_dct_4, &fwd_txfm_2d_cfg_adst_dct_8, + &fwd_txfm_2d_cfg_adst_dct_16, &fwd_txfm_2d_cfg_adst_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_dct_adst_4, &fwd_txfm_2d_cfg_dct_adst_8, + &fwd_txfm_2d_cfg_dct_adst_16, &fwd_txfm_2d_cfg_dct_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &fwd_txfm_2d_cfg_adst_adst_4, &fwd_txfm_2d_cfg_adst_adst_8, + &fwd_txfm_2d_cfg_adst_adst_16, &fwd_txfm_2d_cfg_adst_adst_32 }, }; #endif // CONFIG_EXT_TX
diff --git a/av1/common/av1_inv_txfm2d.c b/av1/common/av1_inv_txfm2d.c index d8dc83f..df772ef 100644 --- a/av1/common/av1_inv_txfm2d.c +++ b/av1/common/av1_inv_txfm2d.c
@@ -52,14 +52,30 @@ }; #else static const TXFM_2D_CFG *inv_txfm_cfg_ls[TX_TYPES][TX_SIZES] = { - { &inv_txfm_2d_cfg_dct_dct_4, &inv_txfm_2d_cfg_dct_dct_8, - &inv_txfm_2d_cfg_dct_dct_16, &inv_txfm_2d_cfg_dct_dct_32 }, - { &inv_txfm_2d_cfg_adst_dct_4, &inv_txfm_2d_cfg_adst_dct_8, - &inv_txfm_2d_cfg_adst_dct_16, &inv_txfm_2d_cfg_adst_dct_32 }, - { &inv_txfm_2d_cfg_dct_adst_4, &inv_txfm_2d_cfg_dct_adst_8, - &inv_txfm_2d_cfg_dct_adst_16, &inv_txfm_2d_cfg_dct_adst_32 }, - { &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, - &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_dct_dct_4, &inv_txfm_2d_cfg_dct_dct_8, + &inv_txfm_2d_cfg_dct_dct_16, &inv_txfm_2d_cfg_dct_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_dct_4, &inv_txfm_2d_cfg_adst_dct_8, + &inv_txfm_2d_cfg_adst_dct_16, &inv_txfm_2d_cfg_adst_dct_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_dct_adst_4, &inv_txfm_2d_cfg_dct_adst_8, + &inv_txfm_2d_cfg_dct_adst_16, &inv_txfm_2d_cfg_dct_adst_32 }, + { +#if CONFIG_CB4X4 + NULL, +#endif + &inv_txfm_2d_cfg_adst_adst_4, &inv_txfm_2d_cfg_adst_adst_8, + &inv_txfm_2d_cfg_adst_adst_16, &inv_txfm_2d_cfg_adst_adst_32 }, }; #endif
diff --git a/av1/common/idct.c b/av1/common/idct.c index 948ac14..1858a0a 100644 --- a/av1/common/idct.c +++ b/av1/common/idct.c
@@ -2511,8 +2511,8 @@ #if CONFIG_CB4X4 void av1_highbd_inv_txfm_add_2x2(const tran_low_t *input, uint8_t *dest, - int stride, int eob, TX_TYPE tx_type, - int lossless, int bd) { + int stride, int eob, int bd, TX_TYPE tx_type, + int lossless) { tran_high_t a1 = input[0] >> UNIT_QUANT_SHIFT; tran_high_t b1 = input[1] >> UNIT_QUANT_SHIFT; tran_high_t c1 = input[2] >> UNIT_QUANT_SHIFT;