Disable filter_intra mode in <8x8 tx blocks 0.159% gain on lowres 60 frames, compared to 0.236% gain if we don't restrict it in small tx blocks. (--disable-ext-partition --disable-ext-partition-types --disable-convolve-round --disable-ext-comp-refs) Change-Id: I1d1c5474ca27de9dec992ea30a9883afd7a56474
diff --git a/av1/common/blockd.h b/av1/common/blockd.h index 452a408..c091422 100644 --- a/av1/common/blockd.h +++ b/av1/common/blockd.h
@@ -249,6 +249,26 @@ static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = { DC_PRED, V_PRED, H_PRED, D117_PRED, D153_PRED, DC_PRED }; + +#define DISABLE_SUB8X8_FILTER_INTRA 1 + +static INLINE int av1_filter_intra_allowed_bsize(BLOCK_SIZE bs) { + (void)bs; +#if DISABLE_SUB8X8_FILTER_INTRA + return block_size_wide[bs] >= 8 && block_size_high[bs] >= 8; +#else + return 1; +#endif +} + +static INLINE int av1_filter_intra_allowed_txsize(TX_SIZE tx) { + (void)tx; +#if DISABLE_SUB8X8_FILTER_INTRA + return tx_size_wide[tx] >= 8 && tx_size_high[tx] >= 8; +#else + return 1; +#endif +} #endif // CONFIG_FILTER_INTRA #if CONFIG_RD_DEBUG
diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index ec5f577..eeaab90 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c
@@ -878,7 +878,8 @@ FILTER_INTRA_MODE_INFO *filter_intra_mode_info = &mbmi->filter_intra_mode_info; - if (mbmi->mode == DC_PRED && mbmi->palette_mode_info.palette_size[0] == 0) { + if (mbmi->mode == DC_PRED && mbmi->palette_mode_info.palette_size[0] == 0 && + av1_filter_intra_allowed_txsize(mbmi->tx_size)) { filter_intra_mode_info->use_filter_intra_mode[0] = aom_read(r, cm->fc->filter_intra_probs[0], ACCT_STR); if (filter_intra_mode_info->use_filter_intra_mode[0]) {
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 128aa76..ea038c5 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -912,7 +912,8 @@ const MACROBLOCKD *xd, const MB_MODE_INFO *const mbmi, aom_writer *w) { - if (mbmi->mode == DC_PRED && mbmi->palette_mode_info.palette_size[0] == 0) { + if (mbmi->mode == DC_PRED && mbmi->palette_mode_info.palette_size[0] == 0 && + av1_filter_intra_allowed_txsize(mbmi->tx_size)) { aom_write(w, mbmi->filter_intra_mode_info.use_filter_intra_mode[0], cm->fc->filter_intra_probs[0]); if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) {
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index 36c8f4b..cb9a0f6 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c
@@ -4159,7 +4159,8 @@ } #if CONFIG_FILTER_INTRA - if (mbmi->mode == DC_PRED && mbmi->palette_mode_info.palette_size[0] == 0) { + if (mbmi->mode == DC_PRED && mbmi->palette_mode_info.palette_size[0] == 0 && + av1_filter_intra_allowed_txsize(mbmi->tx_size)) { const int use_filter_intra_mode = mbmi->filter_intra_mode_info.use_filter_intra_mode[0]; ++counts->filter_intra[0][use_filter_intra_mode];
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 4cd2388..17b3f6f 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c
@@ -2466,6 +2466,12 @@ mbmi->tx_type = tx_type; mbmi->tx_size = tx_size; +#if CONFIG_FILTER_INTRA + if (!is_inter_block(mbmi) && + mbmi->filter_intra_mode_info.use_filter_intra_mode[0] && + !av1_filter_intra_allowed_txsize(tx_size)) + return INT64_MAX; +#endif txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, tx_size, cpi->sf.use_fast_coef_costing); if (rd_stats->rate == INT_MAX) return INT64_MAX; @@ -3285,7 +3291,7 @@ int this_rate; int64_t this_rd, this_model_rd; RD_STATS tokenonly_rd_stats; - if (skip_mask & (1 << mode)) continue; + if (skip_mask & (1 << fimode_to_intradir[mode])) continue; mbmi->filter_intra_mode_info.filter_intra_mode[0] = mode; this_model_rd = intra_model_yrd(cpi, x, bsize, mode_cost); if (*best_model_rd != INT64_MAX && @@ -3735,7 +3741,7 @@ x->palette_y_mode_cost[bsize - BLOCK_8X8][palette_y_mode_ctx][0]; } #if CONFIG_FILTER_INTRA - if (mbmi->mode == DC_PRED) + if (mbmi->mode == DC_PRED && av1_filter_intra_allowed_txsize(mbmi->tx_size)) this_rate += av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0); #endif // CONFIG_FILTER_INTRA #if CONFIG_EXT_INTRA @@ -3784,7 +3790,7 @@ } #if CONFIG_FILTER_INTRA - if (beat_best_rd) { + if (beat_best_rd && av1_filter_intra_allowed_bsize(bsize)) { if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion, skippable, bsize, bmode_costs[DC_PRED], &best_rd, &best_model_rd, @@ -10020,20 +10026,21 @@ skippable = rd_stats_y.skip; #if CONFIG_FILTER_INTRA - int64_t best_rd_tmp = INT64_MAX; - if (rate_y != INT_MAX) { - best_rd_tmp = RDCOST( - x->rdmult, - rate_y + av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0) + - intra_mode_cost[mbmi->mode], - distortion_y); - } if (mbmi->mode == DC_PRED) { RD_STATS rd_stats_y_fi; int filter_intra_selected_flag = 0; TX_SIZE best_tx_size = mbmi->tx_size; TX_TYPE best_tx_type = mbmi->tx_type; FILTER_INTRA_MODE best_fi_mode = FILTER_DC_PRED; + int64_t best_rd_tmp = INT64_MAX; + if (rate_y != INT_MAX && + av1_filter_intra_allowed_txsize(best_tx_size)) { + best_rd_tmp = RDCOST( + x->rdmult, + rate_y + av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 0) + + intra_mode_cost[mbmi->mode], + distortion_y); + } mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 1; for (FILTER_INTRA_MODE fi_mode = FILTER_DC_PRED;