Aggressively prune tx type search at speed 6
Pruned more tx type search at speed 6. Borg test results:
avg_psnr: ovr_psnr: ssim: avg speed change:
hdres: 0.114 0.098 0.191 1.5%
midres: 0.105 0.106 0.145 1.2%
lowres: 0.079 0.060 0.095 3.7%
STATS_CHANGED
Change-Id: I882a9f9f4eda6ef14a241dcdb2a27a04d310aa68
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h
index cc14163..3f6d71c 100644
--- a/av1/encoder/rdopt_utils.h
+++ b/av1/encoder/rdopt_utils.h
@@ -428,12 +428,12 @@
int is_winner_mode) {
// Populate prune transform mode appropriately
txfm_params->prune_2d_txfm_mode = sf->tx_sf.tx_type_search.prune_2d_txfm_mode;
- if (enable_winner_mode_tx_type_pruning) {
- if (is_winner_mode)
- txfm_params->prune_2d_txfm_mode = NO_PRUNE;
- else
- txfm_params->prune_2d_txfm_mode = PRUNE_2D_AGGRESSIVE;
- }
+ if (!enable_winner_mode_tx_type_pruning) return;
+
+ const int prune_mode[2][2] = { { PRUNE_2D_AGGRESSIVE, NO_PRUNE },
+ { PRUNE_2D_MORE_AGGRESSIVE, PRUNE_2D_FAST } };
+ txfm_params->prune_2d_txfm_mode =
+ prune_mode[enable_winner_mode_tx_type_pruning - 1][is_winner_mode];
}
static INLINE void set_tx_domain_dist_params(
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index fdae2ac..fc17776 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -640,6 +640,7 @@
sf->tpl_sf.subpel_force_stop = FULL_PEL;
sf->tpl_sf.disable_filtered_key_tpl = 1;
+ sf->tx_sf.tx_type_search.enable_winner_mode_tx_type_pruning = 2;
sf->tx_sf.use_intra_txb_hash = 1;
sf->tx_sf.tx_type_search.prune_tx_type_est_rd = 0;
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 216ba55..7814ea8b 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -184,6 +184,7 @@
} UENUM1BYTE(MODE_SEARCH_SKIP_LOGIC);
enum {
+ // TODO(yunqing): modify the following names.
NO_PRUNE = 0,
// adaptively prunes the least perspective tx types out of all 16
// (tuned to provide negligible quality loss)
@@ -193,6 +194,7 @@
PRUNE_2D_MORE = 3,
// More aggressive pruning based on tx type score and allowed tx count
PRUNE_2D_AGGRESSIVE = 4,
+ PRUNE_2D_MORE_AGGRESSIVE = 5,
} UENUM1BYTE(TX_TYPE_PRUNE_MODE);
enum {
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index 7b6a46e..d7d0357 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -1639,7 +1639,9 @@
static INLINE float get_adaptive_thresholds(
TX_SIZE tx_size, TxSetType tx_set_type,
TX_TYPE_PRUNE_MODE prune_2d_txfm_mode) {
- const int prune_aggr_table[4][2] = { { 4, 1 }, { 6, 3 }, { 9, 6 }, { 9, 6 } };
+ const int prune_aggr_table[5][2] = {
+ { 4, 1 }, { 6, 3 }, { 9, 6 }, { 9, 6 }, { 12, 9 }
+ };
int pruning_aggressiveness = 0;
if (tx_set_type == EXT_TX_SET_ALL16)
pruning_aggressiveness =
@@ -1812,7 +1814,7 @@
// Enable more pruning based on tx type probability and number of allowed tx
// types
- if (prune_2d_txfm_mode == PRUNE_2D_AGGRESSIVE) {
+ if (prune_2d_txfm_mode >= PRUNE_2D_AGGRESSIVE) {
float temp_score = 0.0;
float score_ratio = 0.0;
int tx_idx, tx_count = 0;
@@ -2033,7 +2035,7 @@
} else {
assert(num_allowed > 0);
int allowed_tx_count =
- (txfm_params->prune_2d_txfm_mode == PRUNE_2D_AGGRESSIVE) ? 1 : 5;
+ (txfm_params->prune_2d_txfm_mode >= PRUNE_2D_AGGRESSIVE) ? 1 : 5;
// !fast_tx_search && txk_end != txk_start && plane == 0
if (txfm_params->prune_2d_txfm_mode >= PRUNE_2D_ACCURATE && is_inter &&
num_allowed > allowed_tx_count) {
