Define separate stage ranges for col/row idtx txfm This will clear range check failures related to fwd square txfms when using --enable-coefficient_range_checking. Change-Id: I5cdc1751b979f612b33d6e00a47277a18792f175
diff --git a/av1/common/av1_inv_txfm1d_cfg.h b/av1/common/av1_inv_txfm1d_cfg.h index 81e5bc2..ba037c2 100644 --- a/av1/common/av1_inv_txfm1d_cfg.h +++ b/av1/common/av1_inv_txfm1d_cfg.h
@@ -29,7 +29,8 @@ static const int8_t inv_stage_range_row_dct_4[4] = { 3, 3, 3, 3 }; static const int8_t inv_stage_range_col_adst_4[6] = { 3, 3, 3, 3, 2, 2 }; static const int8_t inv_stage_range_row_adst_4[6] = { 3, 3, 3, 3, 3, 3 }; -static const int8_t inv_stage_range_idx_4[1] = { 0 }; +static const int8_t inv_stage_range_row_idx_4[1] = { 3 }; +static const int8_t inv_stage_range_col_idx_4[1] = { 3 }; // cos bit static const int8_t inv_cos_bit_col_dct_4[4] = { 13, 13, 13, 13 }; @@ -45,7 +46,8 @@ static const int8_t inv_stage_range_row_dct_8[6] = { 5, 5, 5, 5, 5, 5 }; static const int8_t inv_stage_range_col_adst_8[8] = { 5, 5, 5, 5, 5, 5, 4, 4 }; static const int8_t inv_stage_range_row_adst_8[8] = { 5, 5, 5, 5, 5, 5, 5, 5 }; -static const int8_t inv_stage_range_idx_8[1] = { 0 }; +static const int8_t inv_stage_range_row_idx_8[1] = { 5 }; +static const int8_t inv_stage_range_col_idx_8[1] = { 5 }; // cos bit static const int8_t inv_cos_bit_col_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; @@ -91,7 +93,8 @@ 7, 7, 7, 6, 6 }; static const int8_t inv_stage_range_row_adst_16[10] = { 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }; -static const int8_t inv_stage_range_idx_16[1] = { 0 }; +static const int8_t inv_stage_range_row_idx_16[1] = { 7 }; +static const int8_t inv_stage_range_col_idx_16[1] = { 7 }; // cos bit static const int8_t inv_cos_bit_col_dct_16[8] = { @@ -136,14 +139,15 @@ // stage range static const int8_t inv_stage_range_col_dct_32[10] = { 9, 9, 9, 9, 9, - 9, 9, 9, 8, 8 }; + 9, 9, 9, 9, 9 }; static const int8_t inv_stage_range_row_dct_32[10] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; static const int8_t inv_stage_range_col_adst_32[12] = { 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 8, 8 }; + 9, 9, 9, 9, 9, 9 }; static const int8_t inv_stage_range_row_adst_32[12] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; -static const int8_t inv_stage_range_idx_32[1] = { 0 }; +static const int8_t inv_stage_range_row_idx_32[1] = { 9 }; +static const int8_t inv_stage_range_col_idx_32[1] = { 9 }; // cos bit static const int8_t inv_cos_bit_col_dct_32[10] = { 13, 13, 13, 13, 13, @@ -187,11 +191,12 @@ // stage range static const int8_t inv_stage_range_col_dct_64[12] = { 11, 11, 11, 11, 11, 11, - 11, 11, 11, 11, 10, 10 }; + 11, 11, 11, 11, 11, 11 }; static const int8_t inv_stage_range_row_dct_64[12] = { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }; -static const int8_t inv_stage_range_idx_64[1] = { 0 }; +static const int8_t inv_stage_range_row_idx_64[1] = { 11 }; +static const int8_t inv_stage_range_col_idx_64[1] = { 11 }; // cos bit static const int8_t inv_cos_bit_col_dct_64[12] = { 13, 13, 13, 13, 13, 13, @@ -465,56 +470,105 @@ TXFM_TYPE_ADST32, // .txfm_type }; -// identity does not need to differentiate between row and col -// ---------------- row/col config inv_identity_4 ---------- -static const TXFM_1D_CFG inv_txfm_1d_cfg_identity_4 = { - 4, // .txfm_size - 1, // .stage_num - inv_shift_4, // .shift - inv_stage_range_idx_4, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY4, // .txfm_type +// ---------------- col config inv_identity_4 ---------- +static const TXFM_1D_CFG inv_txfm_1d_col_cfg_identity_4 = { + 4, // .txfm_size + 1, // .stage_num + inv_shift_4, // .shift + inv_stage_range_col_idx_4, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY4, // .txfm_type }; -// ---------------- row/col config inv_identity_8 ---------------- -static const TXFM_1D_CFG inv_txfm_1d_cfg_identity_8 = { - 8, // .txfm_size - 1, // .stage_num - inv_shift_8, // .shift - inv_stage_range_idx_8, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY8, // .txfm_type +// ---------------- row config inv_identity_4 ---------- +static const TXFM_1D_CFG inv_txfm_1d_row_cfg_identity_4 = { + 4, // .txfm_size + 1, // .stage_num + inv_shift_4, // .shift + inv_stage_range_row_idx_4, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY4, // .txfm_type }; -// ---------------- row/col config inv_identity_16 ---------------- -static const TXFM_1D_CFG inv_txfm_1d_cfg_identity_16 = { - 16, // .txfm_size - 1, // .stage_num - inv_shift_16, // .shift - inv_stage_range_idx_16, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY16, // .txfm_type +// ---------------- col config inv_identity_8 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_col_cfg_identity_8 = { + 8, // .txfm_size + 1, // .stage_num + inv_shift_8, // .shift + inv_stage_range_col_idx_8, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY8, // .txfm_type }; -// ---------------- row/col config inv_identity_32 ---------------- -static const TXFM_1D_CFG inv_txfm_1d_cfg_identity_32 = { - 32, // .txfm_size - 1, // .stage_num - inv_shift_32, // .shift - inv_stage_range_idx_32, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY32, // .txfm_type +// ---------------- row config inv_identity_8 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_row_cfg_identity_8 = { + 8, // .txfm_size + 1, // .stage_num + inv_shift_8, // .shift + inv_stage_range_row_idx_8, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY8, // .txfm_type +}; + +// ---------------- col config inv_identity_16 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_col_cfg_identity_16 = { + 16, // .txfm_size + 1, // .stage_num + inv_shift_16, // .shift + inv_stage_range_col_idx_16, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY16, // .txfm_type +}; + +// ---------------- row config inv_identity_16 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_row_cfg_identity_16 = { + 16, // .txfm_size + 1, // .stage_num + inv_shift_16, // .shift + inv_stage_range_row_idx_16, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY16, // .txfm_type +}; + +// ---------------- col config inv_identity_32 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_col_cfg_identity_32 = { + 32, // .txfm_size + 1, // .stage_num + inv_shift_32, // .shift + inv_stage_range_col_idx_32, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY32, // .txfm_type +}; + +// ---------------- row config inv_identity_32 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_row_cfg_identity_32 = { + 32, // .txfm_size + 1, // .stage_num + inv_shift_32, // .shift + inv_stage_range_row_idx_32, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY32, // .txfm_type }; #if CONFIG_TX64X64 -// ---------------- row/col config inv_identity_64 ---------------- -static const TXFM_1D_CFG inv_txfm_1d_cfg_identity_64 = { - 64, // .txfm_size - 1, // .stage_num - inv_shift_64, // .shift - inv_stage_range_idx_64, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY64, // .txfm_type +// ---------------- col config inv_identity_64 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_col_cfg_identity_64 = { + 64, // .txfm_size + 1, // .stage_num + inv_shift_64, // .shift + inv_stage_range_col_idx_64, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY64, // .txfm_type +}; + +// ---------------- row config inv_identity_64 ---------------- +static const TXFM_1D_CFG inv_txfm_1d_row_cfg_identity_64 = { + 64, // .txfm_size + 1, // .stage_num + inv_shift_64, // .shift + inv_stage_range_row_idx_64, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY64, // .txfm_type }; #endif // CONFIG_TX64X64
diff --git a/av1/common/av1_inv_txfm2d.c b/av1/common/av1_inv_txfm2d.c index 84770d7..1874819 100644 --- a/av1/common/av1_inv_txfm2d.c +++ b/av1/common/av1_inv_txfm2d.c
@@ -119,21 +119,21 @@ }, // IDENTITY { - &inv_txfm_1d_cfg_identity_4, &inv_txfm_1d_cfg_identity_8, - &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32, + &inv_txfm_1d_col_cfg_identity_4, &inv_txfm_1d_col_cfg_identity_8, + &inv_txfm_1d_col_cfg_identity_16, &inv_txfm_1d_col_cfg_identity_32, #if CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_64, + &inv_txfm_1d_col_cfg_identity_64, #endif // CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_8, &inv_txfm_1d_cfg_identity_4, - &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_8, - &inv_txfm_1d_cfg_identity_32, &inv_txfm_1d_cfg_identity_16, + &inv_txfm_1d_col_cfg_identity_8, &inv_txfm_1d_col_cfg_identity_4, + &inv_txfm_1d_col_cfg_identity_16, &inv_txfm_1d_col_cfg_identity_8, + &inv_txfm_1d_col_cfg_identity_32, &inv_txfm_1d_col_cfg_identity_16, #if CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_64, &inv_txfm_1d_cfg_identity_32, + &inv_txfm_1d_col_cfg_identity_64, &inv_txfm_1d_col_cfg_identity_32, #endif // CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_4, - &inv_txfm_1d_cfg_identity_32, &inv_txfm_1d_cfg_identity_8, + &inv_txfm_1d_col_cfg_identity_16, &inv_txfm_1d_col_cfg_identity_4, + &inv_txfm_1d_col_cfg_identity_32, &inv_txfm_1d_col_cfg_identity_8, #if CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_64, &inv_txfm_1d_cfg_identity_16, + &inv_txfm_1d_col_cfg_identity_64, &inv_txfm_1d_col_cfg_identity_16, #endif // CONFIG_TX64X64 }, }; @@ -216,21 +216,21 @@ }, // IDENTITY { - &inv_txfm_1d_cfg_identity_4, &inv_txfm_1d_cfg_identity_8, - &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32, + &inv_txfm_1d_row_cfg_identity_4, &inv_txfm_1d_row_cfg_identity_8, + &inv_txfm_1d_row_cfg_identity_16, &inv_txfm_1d_row_cfg_identity_32, #if CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_64, + &inv_txfm_1d_row_cfg_identity_64, #endif // CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_4, &inv_txfm_1d_cfg_identity_8, - &inv_txfm_1d_cfg_identity_8, &inv_txfm_1d_cfg_identity_16, - &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_32, + &inv_txfm_1d_row_cfg_identity_4, &inv_txfm_1d_row_cfg_identity_8, + &inv_txfm_1d_row_cfg_identity_8, &inv_txfm_1d_row_cfg_identity_16, + &inv_txfm_1d_row_cfg_identity_16, &inv_txfm_1d_row_cfg_identity_32, #if CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_32, &inv_txfm_1d_cfg_identity_64, + &inv_txfm_1d_row_cfg_identity_32, &inv_txfm_1d_row_cfg_identity_64, #endif // CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_4, &inv_txfm_1d_cfg_identity_16, - &inv_txfm_1d_cfg_identity_8, &inv_txfm_1d_cfg_identity_32, + &inv_txfm_1d_row_cfg_identity_4, &inv_txfm_1d_row_cfg_identity_16, + &inv_txfm_1d_row_cfg_identity_8, &inv_txfm_1d_row_cfg_identity_32, #if CONFIG_TX64X64 - &inv_txfm_1d_cfg_identity_16, &inv_txfm_1d_cfg_identity_64, + &inv_txfm_1d_row_cfg_identity_16, &inv_txfm_1d_row_cfg_identity_64, #endif // CONFIG_TX64X64 }, };
diff --git a/av1/encoder/av1_fwd_txfm1d_cfg.h b/av1/encoder/av1_fwd_txfm1d_cfg.h index 2788fe9..1d925ed 100644 --- a/av1/encoder/av1_fwd_txfm1d_cfg.h +++ b/av1/encoder/av1_fwd_txfm1d_cfg.h
@@ -23,7 +23,8 @@ static const int8_t fwd_stage_range_row_dct_4[4] = { 2, 3, 3, 3 }; static const int8_t fwd_stage_range_col_adst_4[6] = { 0, 0, 1, 2, 2, 2 }; static const int8_t fwd_stage_range_row_adst_4[6] = { 2, 2, 2, 3, 3, 3 }; -static const int8_t fwd_stage_range_idx_4[1] = { 0 }; +static const int8_t fwd_stage_range_col_idx_4[1] = { 1 }; +static const int8_t fwd_stage_range_row_idx_4[1] = { 2 }; // cos bit static const int8_t fwd_cos_bit_col_dct_4[4] = { 13, 13, 13, 13 }; @@ -40,7 +41,8 @@ static const int8_t fwd_stage_range_row_dct_8[6] = { 3, 4, 5, 5, 5, 5 }; static const int8_t fwd_stage_range_col_adst_8[8] = { 0, 0, 1, 2, 2, 3, 3, 3 }; static const int8_t fwd_stage_range_row_adst_8[8] = { 3, 3, 3, 4, 4, 5, 5, 5 }; -static const int8_t fwd_stage_range_idx_8[1] = { 0 }; +static const int8_t fwd_stage_range_col_idx_8[1] = { 1 }; +static const int8_t fwd_stage_range_row_idx_8[1] = { 4 }; // cos bit static const int8_t fwd_cos_bit_col_dct_8[6] = { 13, 13, 13, 13, 13, 13 }; @@ -64,7 +66,10 @@ static const int8_t fwd_stage_range_row_adst_16[10] = { 4, 4, 4, 5, 5, 6, 6, 7, 7, 7, }; -static const int8_t fwd_stage_range_idx_16[1] = { 0 }; + +static const int8_t fwd_stage_range_col_idx_16[1] = { 2 }; + +static const int8_t fwd_stage_range_row_idx_16[1] = { 6 }; // cos bit static const int8_t fwd_cos_bit_col_dct_16[8] = { @@ -91,7 +96,8 @@ 3, 4, 4, 5, 5, 5 }; static const int8_t fwd_stage_range_row_adst_32[12] = { 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9 }; -static const int8_t fwd_stage_range_idx_32[1] = { 0 }; +static const int8_t fwd_stage_range_col_idx_32[1] = { 2 }; +static const int8_t fwd_stage_range_row_idx_32[1] = { 7 }; // cos bit static const int8_t fwd_cos_bit_col_dct_32[10] = { 12, 12, 12, 12, 12, @@ -112,7 +118,8 @@ 6, 6, 6, 6, 6, 6 }; static const int8_t fwd_stage_range_row_dct_64[12] = { 6, 7, 8, 9, 10, 11, 11, 11, 11, 11, 11, 11 }; -static const int8_t fwd_stage_range_idx_64[1] = { 0 }; +static const int8_t fwd_stage_range_col_idx_64[1] = { 3 }; +static const int8_t fwd_stage_range_row_idx_64[1] = { 9 }; // cos bit static const int8_t fwd_cos_bit_col_dct_64[12] = { 13, 13, 13, 13, 13, 13, @@ -442,56 +449,105 @@ TXFM_TYPE_ADST32, // .txfm_type }; -// identity does not need to differentiate between row and col -// ---------------- row/col config fwd_identity_4 ---------- -static const TXFM_1D_CFG fwd_txfm_1d_cfg_identity_4 = { - 4, // .txfm_size - 1, // .stage_num - fwd_shift_4, // .shift - fwd_stage_range_idx_4, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY4, // .txfm_type +// ---------------- col config fwd_identity_4 ---------- +static const TXFM_1D_CFG fwd_txfm_1d_col_cfg_identity_4 = { + 4, // .txfm_size + 1, // .stage_num + fwd_shift_4, // .shift + fwd_stage_range_col_idx_4, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY4, // .txfm_type }; -// ---------------- row/col config fwd_identity_8 ---------------- -static const TXFM_1D_CFG fwd_txfm_1d_cfg_identity_8 = { - 8, // .txfm_size - 1, // .stage_num - fwd_shift_8, // .shift - fwd_stage_range_idx_8, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY8, // .txfm_type +// ---------------- row config fwd_identity_4 ---------- +static const TXFM_1D_CFG fwd_txfm_1d_row_cfg_identity_4 = { + 4, // .txfm_size + 1, // .stage_num + fwd_shift_4, // .shift + fwd_stage_range_row_idx_4, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY4, // .txfm_type }; -// ---------------- row/col config fwd_identity_16 ---------------- -static const TXFM_1D_CFG fwd_txfm_1d_cfg_identity_16 = { - 16, // .txfm_size - 1, // .stage_num - fwd_shift_16, // .shift - fwd_stage_range_idx_16, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY16, // .txfm_type +// ---------------- col config fwd_identity_8 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_col_cfg_identity_8 = { + 8, // .txfm_size + 1, // .stage_num + fwd_shift_8, // .shift + fwd_stage_range_col_idx_8, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY8, // .txfm_type }; -// ---------------- row/col config fwd_identity_32 ---------------- -static const TXFM_1D_CFG fwd_txfm_1d_cfg_identity_32 = { - 32, // .txfm_size - 1, // .stage_num - fwd_shift_32, // .shift - fwd_stage_range_idx_32, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY32, // .txfm_type +// ---------------- row config fwd_identity_8 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_row_cfg_identity_8 = { + 8, // .txfm_size + 1, // .stage_num + fwd_shift_8, // .shift + fwd_stage_range_row_idx_8, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY8, // .txfm_type +}; + +// ---------------- col config fwd_identity_16 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_col_cfg_identity_16 = { + 16, // .txfm_size + 1, // .stage_num + fwd_shift_16, // .shift + fwd_stage_range_col_idx_16, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY16, // .txfm_type +}; + +// ---------------- row config fwd_identity_16 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_row_cfg_identity_16 = { + 16, // .txfm_size + 1, // .stage_num + fwd_shift_16, // .shift + fwd_stage_range_row_idx_16, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY16, // .txfm_type +}; + +// ---------------- col config fwd_identity_32 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_col_cfg_identity_32 = { + 32, // .txfm_size + 1, // .stage_num + fwd_shift_32, // .shift + fwd_stage_range_col_idx_32, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY32, // .txfm_type +}; + +// ---------------- row config fwd_identity_32 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_row_cfg_identity_32 = { + 32, // .txfm_size + 1, // .stage_num + fwd_shift_32, // .shift + fwd_stage_range_row_idx_32, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY32, // .txfm_type }; #if CONFIG_TX64X64 -// ---------------- row/col config fwd_identity_64 ---------------- -static const TXFM_1D_CFG fwd_txfm_1d_cfg_identity_64 = { - 64, // .txfm_size - 1, // .stage_num - fwd_shift_64, // .shift - fwd_stage_range_idx_64, // .stage_range - NULL, // .cos_bit - TXFM_TYPE_IDENTITY64, // .txfm_type +// ---------------- col config fwd_identity_64 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_col_cfg_identity_64 = { + 64, // .txfm_size + 1, // .stage_num + fwd_shift_64, // .shift + fwd_stage_range_col_idx_64, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY64, // .txfm_type +}; + +// ---------------- row config fwd_identity_64 ---------------- +static const TXFM_1D_CFG fwd_txfm_1d_row_cfg_identity_64 = { + 64, // .txfm_size + 1, // .stage_num + fwd_shift_64, // .shift + fwd_stage_range_row_idx_64, // .stage_range + NULL, // .cos_bit + TXFM_TYPE_IDENTITY64, // .txfm_type }; #endif // CONFIG_TX64X64
diff --git a/av1/encoder/av1_fwd_txfm2d.c b/av1/encoder/av1_fwd_txfm2d.c index 4c58bac..99102f2 100644 --- a/av1/encoder/av1_fwd_txfm2d.c +++ b/av1/encoder/av1_fwd_txfm2d.c
@@ -495,21 +495,21 @@ }, // IDENTITY { - &fwd_txfm_1d_cfg_identity_4, &fwd_txfm_1d_cfg_identity_8, - &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_32, + &fwd_txfm_1d_col_cfg_identity_4, &fwd_txfm_1d_col_cfg_identity_8, + &fwd_txfm_1d_col_cfg_identity_16, &fwd_txfm_1d_col_cfg_identity_32, #if CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_64, + &fwd_txfm_1d_col_cfg_identity_64, #endif // CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_8, &fwd_txfm_1d_cfg_identity_4, - &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_8, - &fwd_txfm_1d_cfg_identity_32, &fwd_txfm_1d_cfg_identity_16, + &fwd_txfm_1d_col_cfg_identity_8, &fwd_txfm_1d_col_cfg_identity_4, + &fwd_txfm_1d_col_cfg_identity_16, &fwd_txfm_1d_col_cfg_identity_8, + &fwd_txfm_1d_col_cfg_identity_32, &fwd_txfm_1d_col_cfg_identity_16, #if CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_64, &fwd_txfm_1d_cfg_identity_32, + &fwd_txfm_1d_col_cfg_identity_64, &fwd_txfm_1d_col_cfg_identity_32, #endif // CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_4, - &fwd_txfm_1d_cfg_identity_32, &fwd_txfm_1d_cfg_identity_8, + &fwd_txfm_1d_col_cfg_identity_16, &fwd_txfm_1d_col_cfg_identity_4, + &fwd_txfm_1d_col_cfg_identity_32, &fwd_txfm_1d_col_cfg_identity_8, #if CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_64, &fwd_txfm_1d_cfg_identity_16, + &fwd_txfm_1d_col_cfg_identity_64, &fwd_txfm_1d_col_cfg_identity_16, #endif // CONFIG_TX64X64 }, }; @@ -592,21 +592,21 @@ }, // IDENTITY { - &fwd_txfm_1d_cfg_identity_4, &fwd_txfm_1d_cfg_identity_8, - &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_32, + &fwd_txfm_1d_row_cfg_identity_4, &fwd_txfm_1d_row_cfg_identity_8, + &fwd_txfm_1d_row_cfg_identity_16, &fwd_txfm_1d_row_cfg_identity_32, #if CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_64, + &fwd_txfm_1d_row_cfg_identity_64, #endif // CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_4, &fwd_txfm_1d_cfg_identity_8, - &fwd_txfm_1d_cfg_identity_8, &fwd_txfm_1d_cfg_identity_16, - &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_32, + &fwd_txfm_1d_row_cfg_identity_4, &fwd_txfm_1d_row_cfg_identity_8, + &fwd_txfm_1d_row_cfg_identity_8, &fwd_txfm_1d_row_cfg_identity_16, + &fwd_txfm_1d_row_cfg_identity_16, &fwd_txfm_1d_row_cfg_identity_32, #if CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_32, &fwd_txfm_1d_cfg_identity_64, + &fwd_txfm_1d_row_cfg_identity_32, &fwd_txfm_1d_row_cfg_identity_64, #endif // CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_4, &fwd_txfm_1d_cfg_identity_16, - &fwd_txfm_1d_cfg_identity_8, &fwd_txfm_1d_cfg_identity_32, + &fwd_txfm_1d_row_cfg_identity_4, &fwd_txfm_1d_row_cfg_identity_16, + &fwd_txfm_1d_row_cfg_identity_8, &fwd_txfm_1d_row_cfg_identity_32, #if CONFIG_TX64X64 - &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_64, + &fwd_txfm_1d_row_cfg_identity_16, &fwd_txfm_1d_row_cfg_identity_64, #endif // CONFIG_TX64X64 }, };
diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc index ead6dbb..ed388db 100644 --- a/test/av1_fwd_txfm2d_test.cc +++ b/test/av1_fwd_txfm2d_test.cc
@@ -142,6 +142,11 @@ param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.6)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.5)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 7)); +#if CONFIG_TX64X64 + if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes. + param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 70, 7)); + } +#endif // CONFIG_TX64X64 param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 2.9, 0.55)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.56)); @@ -157,7 +162,6 @@ #if CONFIG_TX64X64 if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes. - param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 70, 7)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 16, 1.6));
diff --git a/test/av1_inv_txfm2d_test.cc b/test/av1_inv_txfm2d_test.cc index f7ce6b8..bf15d41 100644 --- a/test/av1_inv_txfm2d_test.cc +++ b/test/av1_inv_txfm2d_test.cc
@@ -148,6 +148,11 @@ param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X8, 2, 0.025)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X16, 2, 0.04)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X32, 4, 0.4)); +#if CONFIG_TX64X64 + if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes. + param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X64, 3, 0.2)); + } +#endif // CONFIG_TX64X64 param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X8, 2, 0.016)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.016)); @@ -163,7 +168,6 @@ #if CONFIG_TX64X64 if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes. - param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X64, 3, 0.2)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X64, 3, 0.3)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X32, 3, 0.31)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.16));