Move rect txfm's shift setting into config For 2:1 txfm, move the sqrt2 scaling to the end of the fwd txfm function This only affects the txfm accuracy slightly but simplifies the range management a lot. Change-Id: I2eef8c15b7e8958a1fb0f2ae632604fdbab86046
diff --git a/av1/encoder/av1_fwd_txfm1d_cfg.h b/av1/encoder/av1_fwd_txfm1d_cfg.h index 8673113..097ef96 100644 --- a/av1/encoder/av1_fwd_txfm1d_cfg.h +++ b/av1/encoder/av1_fwd_txfm1d_cfg.h
@@ -213,7 +213,7 @@ 10, 10, 10, 10, 10, 10 }; // ---------------- 4x16 1D constants ----------------------- -#define fwd_shift_4x16 fwd_shift_16 +static const int8_t fwd_shift_4x16[3] = { 2, -1, 0 }; static const int8_t fwd_stage_range_row_dct_4x16[4] = { 4, 5, 5, 5 }; static const int8_t fwd_stage_range_row_adst_4x16[6] = { 4, 4, 4, 5, 5, 5 }; static const int8_t fwd_stage_range_row_idx_4x16[1] = { 5 }; @@ -221,7 +221,7 @@ static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 12, 12, 12, 12, 12, 12 }; // ---------------- 16x4 1D constants ----------------------- -#define fwd_shift_16x4 fwd_shift_16 +static const int8_t fwd_shift_16x4[3] = { 2, -1, 0 }; static const int8_t fwd_stage_range_row_dct_16x4[8] = { 2, 3, 4, 5, 5, 5, 5, 5 }; @@ -234,7 +234,7 @@ 12, 12, 12, 12, 12 }; // ---------------- 8x32 1D constants ----------------------- -#define fwd_shift_8x32 fwd_shift_32 +static const int8_t fwd_shift_8x32[3] = { 2, -2, 0 }; static const int8_t fwd_stage_range_row_dct_8x32[6] = { 5, 6, 7, 7, 7, 7 }; static const int8_t fwd_stage_range_row_adst_8x32[8] = { 5, 5, 5, 6, 6, 7, 7, 7 @@ -245,7 +245,7 @@ 11, 11, 11, 11 }; // ---------------- 32x8 1D constants ----------------------- -#define fwd_shift_32x8 fwd_shift_32 +static const int8_t fwd_shift_32x8[3] = { 2, -2, 0 }; static const int8_t fwd_stage_range_row_dct_32x8[10] = { 3, 4, 5, 6, 7, 7, 7, 7, 7, 7 }; static const int8_t fwd_stage_range_row_adst_32x8[12] = { 3, 3, 3, 4, 4, 5, @@ -257,7 +257,7 @@ 12, 11, 11, 11, 11, 11 }; // ---------------- 16x64 1D constants ----------------------- -#define fwd_shift_16x64 fwd_shift_64 +static const int8_t fwd_shift_16x64[3] = { 0, -2, 0 }; static const int8_t fwd_stage_range_row_dct_16x64[8] = { 6, 7, 8, 9, 9, 9, 9, 9 }; @@ -266,7 +266,7 @@ 10, 10, 10, 10 }; // ---------------- 64x16 1D constants ----------------------- -#define fwd_shift_64x16 fwd_shift_64 +static const int8_t fwd_shift_64x16[3] = { 0, -2, 0 }; static const int8_t fwd_stage_range_row_dct_64x16[12] = { 4, 5, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9 }; static const int8_t fwd_stage_range_row_idx_64x16[1] = { 7 };
diff --git a/av1/encoder/av1_fwd_txfm2d.c b/av1/encoder/av1_fwd_txfm2d.c index 176dd41..ef51871 100644 --- a/av1/encoder/av1_fwd_txfm2d.c +++ b/av1/encoder/av1_fwd_txfm2d.c
@@ -47,45 +47,17 @@ void av1_gen_fwd_stage_range(int8_t *stage_range_col, int8_t *stage_range_row, const TXFM_2D_FLIP_CFG *cfg, int bd) { - // Note when assigning txfm_size_col, we use the txfm_size from the - // row configuration and vice versa. This is intentionally done to - // accurately perform rectangular transforms. When the transform is - // rectangular, the number of columns will be the same as the - // txfm_size stored in the row cfg struct. It will make no difference - // for square transforms. - const int txfm_size_col = cfg->row_cfg->txfm_size; - const int txfm_size_row = cfg->col_cfg->txfm_size; // Take the shift from the larger dimension in the rectangular case. - const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift - : cfg->col_cfg->shift; + const int8_t *shift = cfg->row_cfg->shift; // i < MAX_TXFM_STAGE_NUM will mute above array bounds warning for (int i = 0; i < cfg->col_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) { stage_range_col[i] = cfg->col_cfg->stage_range[i] + shift[0] + bd + 1; } - const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - int rect_type_shift = 0; - - int shift2 = shift[2]; - if (rect_type == 1 || rect_type == -1) { - rect_type_shift = 1; - } else if (rect_type == 2 || rect_type == -2) { - const int txfm_size_max = AOMMAX(txfm_size_col, txfm_size_row); - - // For 64x16 / 16x64 / 32x8 / 8x32 shift 2 bits, and - // For 16x4 / 4x16 shift by 1 bit. - rect_type_shift = (txfm_size_max >= 32) ? 2 : 1; - } - - while (rect_type_shift > 0 && shift2 < 0) { - shift2++; - rect_type_shift--; - } - // i < MAX_TXFM_STAGE_NUM will mute above array bounds warning for (int i = 0; i < cfg->row_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) { - stage_range_row[i] = cfg->row_cfg->stage_range[i] + shift[0] + shift[1] + - bd + 1 + rect_type_shift; + stage_range_row[i] = + cfg->row_cfg->stage_range[i] + shift[0] + shift[1] + bd + 1; } } @@ -102,28 +74,8 @@ const int txfm_size_col = cfg->row_cfg->txfm_size; const int txfm_size_row = cfg->col_cfg->txfm_size; // Take the shift from the larger dimension in the rectangular case. - const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift - : cfg->col_cfg->shift; - int shift2 = shift[2]; + const int8_t *shift = cfg->row_cfg->shift; const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row); - int rect_type1_shift = 0; - int rect_type2_shift = 0; - if (rect_type == 1 || rect_type == -1) { - rect_type1_shift = 1; - while (rect_type1_shift > 0 && shift2 < 0) { - shift2++; - rect_type1_shift--; - } - } else if (rect_type == 2 || rect_type == -2) { - const int txfm_size_max = AOMMAX(txfm_size_col, txfm_size_row); - // For 64x16 / 16x64 / 32x8 / 8x32 shift 2 bits, and - // For 16x4 / 4x16 shift by 1 bit. - rect_type2_shift = (txfm_size_max >= 32) ? 2 : 1; - while (rect_type2_shift > 0 && shift2 < 0) { - shift2++; - rect_type2_shift--; - } - } int8_t stage_range_col[MAX_TXFM_STAGE_NUM]; int8_t stage_range_row[MAX_TXFM_STAGE_NUM]; assert(cfg->col_cfg->stage_num <= MAX_TXFM_STAGE_NUM); @@ -150,16 +102,6 @@ } av1_round_shift_array(temp_in, txfm_size_row, -shift[0]); txfm_func_col(temp_in, temp_out, cos_bit_col, stage_range_col); - // Multiply everything by Sqrt2 on the larger dimension if the - // transform is rectangular and the size difference is a factor of 2. - // If the size difference is a factor of 4, multiply by - // 2^rect_type_2_extra_shift. - if (abs(rect_type) == 1 && rect_type1_shift == 1) { - for (r = 0; r < txfm_size_row; ++r) - temp_out[r] = (int32_t)fdct_round_shift(temp_out[r] * Sqrt2); - } else if (abs(rect_type) == 2) { - av1_round_shift_array(temp_out, txfm_size_row, -rect_type2_shift); - } av1_round_shift_array(temp_out, txfm_size_row, -shift[1]); if (cfg->lr_flip == 0) { for (r = 0; r < txfm_size_row; ++r) @@ -175,12 +117,14 @@ for (r = 0; r < txfm_size_row; ++r) { txfm_func_row(buf + r * txfm_size_col, output + r * txfm_size_col, cos_bit_row, stage_range_row); - if (abs(rect_type) == 1 && rect_type1_shift == 0) { + if (abs(rect_type) == 1) { + // Multiply everything by Sqrt2 if the transform is rectangular and the + // size difference is a factor of 2. for (c = 0; c < txfm_size_col; ++c) output[r * txfm_size_col + c] = - (int32_t)fdct_round_shift(output[r * txfm_size_col + c] * InvSqrt2); + (int32_t)fdct_round_shift(output[r * txfm_size_col + c] * Sqrt2); } - av1_round_shift_array(output + r * txfm_size_col, txfm_size_col, -shift2); + av1_round_shift_array(output + r * txfm_size_col, txfm_size_col, -shift[2]); } }
diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc index c12279a..2ab629f 100644 --- a/test/av1_fwd_txfm2d_test.cc +++ b/test/av1_fwd_txfm2d_test.cc
@@ -149,11 +149,11 @@ #endif // CONFIG_TX64X64 param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.58)); - param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.58)); + param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.6, 0.63)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 15, 1.5)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 15, 1.5)); - param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 55, 7)); - param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 30, 7)); + param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 57, 7)); + param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 7)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 5, 0.7)); param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5.5, 0.9));
diff --git a/test/av1_inv_txfm2d_test.cc b/test/av1_inv_txfm2d_test.cc index f46c856..9866eb5 100644 --- a/test/av1_inv_txfm2d_test.cc +++ b/test/av1_inv_txfm2d_test.cc
@@ -155,11 +155,11 @@ #endif // CONFIG_TX64X64 param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X8, 2, 0.016)); - param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.016)); + param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.025)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X16, 2, 0.2)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X8, 2, 0.2)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X32, 3, 0.4)); - param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X16, 3, 0.4)); + param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X16, 3, 0.5)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X16, 2, 0.2)); param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X4, 2, 0.2));
diff --git a/test/av1_txfm_test.cc b/test/av1_txfm_test.cc index e7b4b22..be5e756 100644 --- a/test/av1_txfm_test.cc +++ b/test/av1_txfm_test.cc
@@ -214,9 +214,7 @@ av1_get_fwd_txfm_cfg(tx_type, tx_size, &fwd_txfm_flip_cfg); const int tx_width = fwd_txfm_flip_cfg.row_cfg->txfm_size; const int tx_height = fwd_txfm_flip_cfg.col_cfg->txfm_size; - const int8_t *shift = (tx_width > tx_height) - ? fwd_txfm_flip_cfg.row_cfg->shift - : fwd_txfm_flip_cfg.col_cfg->shift; + const int8_t *shift = fwd_txfm_flip_cfg.row_cfg->shift; const int amplify_bit = shift[0] + shift[1] + shift[2]; double amplify_factor = amplify_bit >= 0 ? (1 << amplify_bit) : (1.0 / (1 << -amplify_bit)); @@ -225,10 +223,6 @@ const int rect_type = get_rect_tx_log_ratio(tx_width, tx_height); if (abs(rect_type) == 1) { amplify_factor *= pow(2, 0.5); - } else if (abs(rect_type) == 2) { - const int tx_max_dim = AOMMAX(tx_width, tx_height); - const int rect_type2_shift = (tx_max_dim >= 32) ? 2 : 1; - amplify_factor *= pow(2, rect_type2_shift); } return amplify_factor; }