CONFIG_IST_REDUCTION Search only 4 IST sets at encoder
diff --git a/aom_dsp/txfm_common.h b/aom_dsp/txfm_common.h index 5731e92..6a51664 100644 --- a/aom_dsp/txfm_common.h +++ b/aom_dsp/txfm_common.h
@@ -28,6 +28,9 @@ // Primary transform set used for the current tx block. TX_TYPE tx_type; #if CONFIG_IST_SET_FLAG +#if CONFIG_IST_REDUCTION + int sec_tx_set_idx; +#endif // CONFIG_IST_REDUCTION // for both forward and inverse secondary transforms // Secondary transform set used for the current tx block. TX_TYPE sec_tx_set;
diff --git a/av1/common/common_data.h b/av1/common/common_data.h index 27e8476..047bca4 100644 --- a/av1/common/common_data.h +++ b/av1/common/common_data.h
@@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2021, Alliance for Open Media. All rights reserved * * This source code is subject to the terms of the BSD 3-Clause Clear License @@ -1014,6 +1014,28 @@ block_size_high[bsize1] > block_size_high[bsize2]; } +#if CONFIG_IST_REDUCTION +static const uint8_t ist_intra_stx_mapping[IST_DIR_SIZE][IST_DIR_SIZE] = { + { 6, 1, 0, 5, 4, 3, 2 }, // DC_PRED + { 1, 6, 0, 4, 2, 5, 3 }, // V_PRED, H_PRED, SMOOTH_V_PRED， SMOOTH_H_PRED + { 2, 6, 0, 5, 1, 4, 3 }, // D45_PRED + { 3, 4, 6, 1, 0, 2, 5 }, // D135_PRED + { 4, 1, 3, 6, 0, 5, 2 }, // D113_PRED, D157_PRED + { 5, 0, 6, 2, 1, 4, 3 }, // D203_PRED, D67_PRED + { 6, 1, 0, 5, 4, 3, 2 }, // SMOOTH_PRED +}; + +static const uint8_t inv_ist_intra_stx_mapping[IST_DIR_SIZE][IST_DIR_SIZE] = { + { 2, 1, 6, 5, 4, 3, 0 }, // DC_PRED + { 2, 0, 4, 6, 3, 5, 1 }, // V_PRED, H_PRED, SMOOTH_V_PRED， SMOOTH_H_PRED + { 2, 4, 0, 6, 5, 3, 1 }, // D45_PRED + { 4, 3, 5, 0, 1, 6, 2 }, // D135_PRED + { 4, 1, 6, 2, 0, 5, 3 }, // D113_PRED, D157_PRED + { 1, 4, 3, 6, 5, 0, 2 }, // D203_PRED, D67_PRED + { 2, 1, 6, 5, 4, 3, 0 }, // SMOOTH_PRED +}; +#endif // CONFIG_IST_REDUCTION + #if CONFIG_INTRA_TX_IST_PARSE // Mapping of IST kernel set to an index based on intra mode. // The index will be signaled in the bitstream
diff --git a/av1/common/enums.h b/av1/common/enums.h index 701a618..60eb51c 100644 --- a/av1/common/enums.h +++ b/av1/common/enums.h
@@ -28,6 +28,11 @@ /*!\cond */ #undef MAX_SB_SIZE + +#if CONFIG_IST_REDUCTION +#define IST_REDUCE_SET_SIZE 4 +#endif + #define BAWP_BUGFIX 1 #define ADJUST_SUPER_RES_Q 1
diff --git a/av1/common/idct.c b/av1/common/idct.c index 72ff447..c55a804 100644 --- a/av1/common/idct.c +++ b/av1/common/idct.c
@@ -431,6 +431,9 @@ txfm_param->tx_type = get_primary_tx_type(tx_type); #if CONFIG_IST_SET_FLAG txfm_param->sec_tx_set = 0; +#if CONFIG_IST_REDUCTION + txfm_param->sec_tx_set_idx = 0; +#endif // CONFIG_IST_REDUCTION #endif // CONFIG_IST_SET_FLAG txfm_param->sec_tx_type = 0; txfm_param->intra_mode = get_intra_mode(mbmi, plane); @@ -447,6 +450,17 @@ txfm_param->sec_tx_type = get_secondary_tx_type(tx_type); #if CONFIG_IST_SET_FLAG txfm_param->sec_tx_set = get_secondary_tx_set(tx_type); +#if CONFIG_IST_REDUCTION + uint8_t intra_stx_mode = stx_transpose_mapping[txfm_param->intra_mode]; + uint8_t stx_id; + if (txfm_param->tx_type == ADST_ADST) { + stx_id = txfm_param->sec_tx_set - IST_DIR_SIZE; + } else { + stx_id = txfm_param->sec_tx_set; + } + uint8_t stx_idx = inv_ist_intra_stx_mapping[intra_stx_mode][stx_id]; + txfm_param->sec_tx_set_idx = stx_idx; +#endif // CONFIG_IST_REDUCTION #endif // CONFIG_IST_SET_FLAG } txfm_param->tx_size = tx_size;
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index e45ed98..f683a92 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c
@@ -527,7 +527,12 @@ // per set) Set 0 ~ IST_DIR_SIZE-1 for DCT_DCT, and Set IST_DIR_SIZE ~ // IST_SET_SIZE-1 for ADST_ADST if (txfm_param->sec_tx_type == 0 && - (txfm_param->sec_tx_set == 0 || txfm_param->sec_tx_set == IST_DIR_SIZE)) +#if CONFIG_IST_REDUCTION + txfm_param->sec_tx_set_idx == 0 +#else + (txfm_param->sec_tx_set == 0 || txfm_param->sec_tx_set == IST_DIR_SIZE) +#endif // CONFIG_IST_REDUCTION + ) #else if (txfm_param->sec_tx_type == 0) #endif // CONFIG_IST_ANY_SET @@ -649,6 +654,9 @@ txfm_param->tx_type = get_primary_tx_type(tx_type); #if CONFIG_IST_SET_FLAG txfm_param->sec_tx_set = 0; +#if CONFIG_IST_REDUCTION + txfm_param->sec_tx_set_idx = 0; +#endif // CONFIG_IST_REDUCTION #endif // CONFIG_IST_SET_FLAG txfm_param->sec_tx_type = 0; txfm_param->intra_mode = get_intra_mode(mbmi, plane); @@ -666,6 +674,17 @@ !(mbmi->fsc_mode[xd->tree_type == CHROMA_PART])) { #if CONFIG_IST_SET_FLAG txfm_param->sec_tx_set = get_secondary_tx_set(tx_type); +#if CONFIG_IST_REDUCTION + uint8_t intra_stx_mode = stx_transpose_mapping[txfm_param->intra_mode]; + uint8_t stx_id; + if (txfm_param->tx_type == ADST_ADST) { + stx_id = txfm_param->sec_tx_set - IST_DIR_SIZE; + } else { + stx_id = txfm_param->sec_tx_set; + } + uint8_t stx_idx = inv_ist_intra_stx_mapping[intra_stx_mode][stx_id]; + txfm_param->sec_tx_set_idx = stx_idx; +#endif // CONFIG_IST_REDUCTION #endif // CONFIG_IST_SET_FLAG txfm_param->sec_tx_type = get_secondary_tx_type(tx_type); }
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c index 260d98a..b4ad627 100644 --- a/av1/encoder/tx_search.c +++ b/av1/encoder/tx_search.c
@@ -2843,15 +2843,37 @@ int max_set_id = (skip_stx || is_inter_block(mbmi, xd->tree_type)) ? 1 : IST_DIR_SIZE; #endif +#if CONFIG_IST_REDUCTION + if (max_set_id == IST_DIR_SIZE) { + max_set_id = IST_REDUCE_SET_SIZE; + // if (txw <= 4 || txh <= 4){ + // max_set_id = 1; + // } + } + for (int set_idx = init_set_id; set_idx < max_set_id; ++set_idx) { + txfm_param.sec_tx_set_idx = set_idx; + uint8_t set_id = set_idx; + if (!is_inter_block(mbmi, xd->tree_type)) { + const PREDICTION_MODE mode = AOMMIN(intra_mode, SMOOTH_H_PRED); + int intra_stx_mode = stx_transpose_mapping[mode]; + assert(set_idx < IST_REDUCE_SET_SIZE); + set_id = ist_intra_stx_mapping[intra_stx_mode][set_idx]; + } +#else // Iterate through all possible secondary tx sets for given primary tx type for (int set_id = init_set_id; set_id < max_set_id; ++set_id) { +#endif // CONFIG_IST_REDUCTION #endif // CONFIG_IST_ANY_SET const int max_stx = xd->enable_ist && !(eob_found) ? 4 : 1; for (int stx = 0; stx < max_stx; ++stx) { // Skip repeated evaluation of no secondary transform. +#if CONFIG_IST_REDUCTION + if (set_idx && !stx) continue; +#else if (set_id && !stx) continue; +#endif // CONFIG_IST_REDUCTION #if CONFIG_IST_ANY_SET TX_TYPE tx_type = primary_tx_type;
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake index bc3d3de..62f8c11 100644 --- a/build/cmake/aom_config_defaults.cmake +++ b/build/cmake/aom_config_defaults.cmake
@@ -159,6 +159,9 @@ #IST multiset for inter set_aom_config_var(CONFIG_IST_INTER_MULTISET 1 "Enable multiset IST for inter TU.") +#IST set reduction (non-normative) +set_aom_config_var(CONFIG_IST_REDUCTION 1 + "Use 4 sets for IST encoder search.") # AV2 experiment flags. set_aom_config_var(CONFIG_IMPROVEIDTX 1