Divide handle_intra_mode to handle luma and chroma separately
BUG=aomedia:2863
Change-Id: Icd037cefdf58864c2e6ecd4db416665b2c3940a4
diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index d50f8ea..2f621d8 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -846,6 +846,139 @@
}
}
+static INLINE int handle_intra_y_mode(IntraModeSearchState *intra_search_state,
+ const AV1_COMP *cpi, MACROBLOCK *x,
+ BLOCK_SIZE bsize,
+ unsigned int ref_frame_cost,
+ const PICK_MODE_CONTEXT *ctx,
+ RD_STATS *rd_stats_y, int64_t best_rd) {
+ const AV1_COMMON *cm = &cpi->common;
+ const SPEED_FEATURES *const sf = &cpi->sf;
+ MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *const mbmi = xd->mi[0];
+ assert(mbmi->ref_frame[0] == INTRA_FRAME);
+ const PREDICTION_MODE mode = mbmi->mode;
+ const ModeCosts *mode_costs = &x->mode_costs;
+ const int mode_cost =
+ mode_costs->mbmode_cost[size_group_lookup[bsize]][mode] + ref_frame_cost;
+
+ const int is_directional_mode = av1_is_directional_mode(mode);
+ if (is_directional_mode && av1_use_angle_delta(bsize) &&
+ cpi->oxcf.intra_mode_cfg.enable_angle_delta) {
+ if (sf->intra_sf.intra_pruning_with_hog &&
+ !intra_search_state->dir_mode_skip_mask_ready) {
+ prune_intra_mode_with_hog(x, bsize,
+ cpi->sf.intra_sf.intra_pruning_with_hog_thresh,
+ intra_search_state->directional_mode_skip_mask);
+ intra_search_state->dir_mode_skip_mask_ready = 1;
+ }
+ if (intra_search_state->directional_mode_skip_mask[mode]) return 0;
+ av1_init_rd_stats(rd_stats_y);
+ rd_stats_y->rate = INT_MAX;
+ int64_t model_rd = INT64_MAX;
+ int rate_dummy;
+ rd_pick_intra_angle_sby(cpi, x, &rate_dummy, rd_stats_y, bsize, mode_cost,
+ best_rd, &model_rd, 0);
+
+ } else {
+ av1_init_rd_stats(rd_stats_y);
+ mbmi->angle_delta[PLANE_TYPE_Y] = 0;
+ av1_pick_uniform_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, best_rd);
+ }
+
+ // Pick filter intra modes.
+ if (mode == DC_PRED && av1_filter_intra_allowed_bsize(cm, bsize)) {
+ int try_filter_intra = 1;
+ int64_t best_rd_so_far = INT64_MAX;
+ if (rd_stats_y->rate != INT_MAX) {
+ const int tmp_rate = rd_stats_y->rate +
+ mode_costs->filter_intra_cost[bsize][0] + mode_cost;
+ best_rd_so_far = RDCOST(x->rdmult, tmp_rate, rd_stats_y->dist);
+ try_filter_intra = (best_rd_so_far / 2) <= best_rd;
+ }
+
+ if (try_filter_intra) {
+ handle_filter_intra_mode(cpi, x, bsize, ctx, rd_stats_y, mode_cost,
+ best_rd, best_rd_so_far);
+ }
+ }
+
+ if (rd_stats_y->rate == INT_MAX) return 0;
+
+ return 1;
+}
+
+static INLINE int handle_intra_uv_mode(IntraModeSearchState *intra_search_state,
+ const AV1_COMP *cpi, MACROBLOCK *x,
+ BLOCK_SIZE bsize, RD_STATS *rd_stats,
+ const RD_STATS *rd_stats_y,
+ RD_STATS *rd_stats_uv, int64_t best_rd,
+ int mode_cost_y) {
+ const AV1_COMMON *cm = &cpi->common;
+ MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *const mbmi = xd->mi[0];
+ assert(mbmi->ref_frame[0] == INTRA_FRAME);
+ const ModeCosts *mode_costs = &x->mode_costs;
+ const int skip_ctx = av1_get_skip_txfm_context(xd);
+
+ // TODO(chiyotsai@google.com): Consolidate the chroma search code here with
+ // the one in av1_search_palette_mode.
+ PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
+ const int try_palette =
+ cpi->oxcf.tool_cfg.enable_palette &&
+ av1_allow_palette(cm->features.allow_screen_content_tools, mbmi->bsize);
+
+ if (intra_search_state->rate_uv_intra == INT_MAX) {
+ // If no good uv-predictor had been found, search for it.
+ const int rate_y = rd_stats_y->skip_txfm
+ ? mode_costs->skip_txfm_cost[skip_ctx][1]
+ : rd_stats_y->rate;
+ const int64_t rdy =
+ RDCOST(x->rdmult, rate_y + mode_cost_y, rd_stats_y->dist);
+ if (best_rd < (INT64_MAX / 2) && rdy > (best_rd + (best_rd >> 2))) {
+ intra_search_state->skip_intra_modes = 1;
+ return 0;
+ }
+ const TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd);
+ av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra,
+ &intra_search_state->rate_uv_tokenonly,
+ &intra_search_state->dist_uvs,
+ &intra_search_state->skip_uvs, bsize, uv_tx);
+ intra_search_state->mode_uv = mbmi->uv_mode;
+ if (try_palette) intra_search_state->pmi_uv = *pmi;
+ intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
+
+ const int uv_rate = intra_search_state->rate_uv_tokenonly;
+ const int64_t uv_dist = intra_search_state->dist_uvs;
+ const int64_t uv_rd = RDCOST(x->rdmult, uv_rate, uv_dist);
+ if (uv_rd > best_rd) {
+ // If there is no good intra uv-mode available, we can skip all intra
+ // modes.
+ intra_search_state->skip_intra_modes = 1;
+ return 0;
+ }
+ }
+
+ // If we are here, then the encoder has found at least one good intra uv
+ // predictor, so we can directly copy its statistics over.
+ // TODO(any): the stats here is probably not right if the current best mode
+ // is cfl.
+ rd_stats_uv->rate = intra_search_state->rate_uv_tokenonly;
+ rd_stats_uv->dist = intra_search_state->dist_uvs;
+ rd_stats_uv->skip_txfm = intra_search_state->skip_uvs;
+ rd_stats->skip_txfm = rd_stats_y->skip_txfm && rd_stats_uv->skip_txfm;
+ mbmi->uv_mode = intra_search_state->mode_uv;
+ if (try_palette) {
+ pmi->palette_size[1] = intra_search_state->pmi_uv.palette_size[1];
+ memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
+ intra_search_state->pmi_uv.palette_colors + PALETTE_MAX_SIZE,
+ 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0]));
+ }
+ mbmi->angle_delta[PLANE_TYPE_UV] = intra_search_state->uv_angle_delta;
+
+ return 1;
+}
+
int64_t av1_handle_intra_mode(IntraModeSearchState *intra_search_state,
const AV1_COMP *cpi, MACROBLOCK *x,
BLOCK_SIZE bsize, unsigned int ref_frame_cost,
@@ -877,108 +1010,26 @@
return INT64_MAX;
}
- const int is_directional_mode = av1_is_directional_mode(mode);
- if (is_directional_mode && av1_use_angle_delta(bsize) &&
- cpi->oxcf.intra_mode_cfg.enable_angle_delta) {
- if (sf->intra_sf.intra_pruning_with_hog &&
- !intra_search_state->dir_mode_skip_mask_ready) {
- prune_intra_mode_with_hog(x, bsize,
- cpi->sf.intra_sf.intra_pruning_with_hog_thresh,
- intra_search_state->directional_mode_skip_mask);
- intra_search_state->dir_mode_skip_mask_ready = 1;
- }
- if (intra_search_state->directional_mode_skip_mask[mode]) return INT64_MAX;
- av1_init_rd_stats(rd_stats_y);
- rd_stats_y->rate = INT_MAX;
- int64_t model_rd = INT64_MAX;
- int rate_dummy;
- rd_pick_intra_angle_sby(cpi, x, &rate_dummy, rd_stats_y, bsize, mode_cost,
- best_rd, &model_rd, 0);
+ const int intra_y_mode_valid =
+ handle_intra_y_mode(intra_search_state, cpi, x, bsize, ref_frame_cost,
+ ctx, rd_stats_y, best_rd);
- } else {
- av1_init_rd_stats(rd_stats_y);
- mbmi->angle_delta[PLANE_TYPE_Y] = 0;
- av1_pick_uniform_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, best_rd);
+ if (!intra_y_mode_valid) {
+ return INT64_MAX;
}
- // Pick filter intra modes.
- if (mode == DC_PRED && av1_filter_intra_allowed_bsize(cm, bsize)) {
- int try_filter_intra = 1;
- int64_t best_rd_so_far = INT64_MAX;
- if (rd_stats_y->rate != INT_MAX) {
- const int tmp_rate = rd_stats_y->rate +
- mode_costs->filter_intra_cost[bsize][0] + mode_cost;
- best_rd_so_far = RDCOST(x->rdmult, tmp_rate, rd_stats_y->dist);
- try_filter_intra = (best_rd_so_far / 2) <= best_rd;
- }
-
- if (try_filter_intra) {
- handle_filter_intra_mode(cpi, x, bsize, ctx, rd_stats_y, mode_cost,
- best_rd, best_rd_so_far);
- }
- }
-
- if (rd_stats_y->rate == INT_MAX) return INT64_MAX;
-
const int mode_cost_y =
intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost);
av1_init_rd_stats(rd_stats);
av1_init_rd_stats(rd_stats_uv);
const int num_planes = av1_num_planes(cm);
if (num_planes > 1) {
- // TODO(chiyotsai@google.com): Consolidate the chroma search code here with
- // the one in av1_search_palette_mode.
- PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
- const int try_palette =
- cpi->oxcf.tool_cfg.enable_palette &&
- av1_allow_palette(cm->features.allow_screen_content_tools, mbmi->bsize);
- if (intra_search_state->rate_uv_intra == INT_MAX) {
- // If no good uv-predictor had been found, search for it.
- const int rate_y = rd_stats_y->skip_txfm
- ? mode_costs->skip_txfm_cost[skip_ctx][1]
- : rd_stats_y->rate;
- const int64_t rdy =
- RDCOST(x->rdmult, rate_y + mode_cost_y, rd_stats_y->dist);
- if (best_rd < (INT64_MAX / 2) && rdy > (best_rd + (best_rd >> 2))) {
- intra_search_state->skip_intra_modes = 1;
- return INT64_MAX;
- }
- const TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd);
- av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra,
- &intra_search_state->rate_uv_tokenonly,
- &intra_search_state->dist_uvs,
- &intra_search_state->skip_uvs, bsize, uv_tx);
- intra_search_state->mode_uv = mbmi->uv_mode;
- if (try_palette) intra_search_state->pmi_uv = *pmi;
- intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
-
- const int uv_rate = intra_search_state->rate_uv_tokenonly;
- const int64_t uv_dist = intra_search_state->dist_uvs;
- const int64_t uv_rd = RDCOST(x->rdmult, uv_rate, uv_dist);
- if (uv_rd > best_rd) {
- // If there is no good intra uv-mode available, we can skip all intra
- // modes.
- intra_search_state->skip_intra_modes = 1;
- return INT64_MAX;
- }
+ const int intra_uv_mode_valid =
+ handle_intra_uv_mode(intra_search_state, cpi, x, bsize, rd_stats,
+ rd_stats_y, rd_stats_uv, best_rd, mode_cost_y);
+ if (!intra_uv_mode_valid) {
+ return INT64_MAX;
}
-
- // If we are here, then the encoder has found at least one good intra uv
- // predictor, so we can directly copy its statistics over.
- // TODO(any): the stats here is probably not right if the current best mode
- // is cfl.
- rd_stats_uv->rate = intra_search_state->rate_uv_tokenonly;
- rd_stats_uv->dist = intra_search_state->dist_uvs;
- rd_stats_uv->skip_txfm = intra_search_state->skip_uvs;
- rd_stats->skip_txfm = rd_stats_y->skip_txfm && rd_stats_uv->skip_txfm;
- mbmi->uv_mode = intra_search_state->mode_uv;
- if (try_palette) {
- pmi->palette_size[1] = intra_search_state->pmi_uv.palette_size[1];
- memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
- intra_search_state->pmi_uv.palette_colors + PALETTE_MAX_SIZE,
- 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0]));
- }
- mbmi->angle_delta[PLANE_TYPE_UV] = intra_search_state->uv_angle_delta;
}
rd_stats->rate = rd_stats_y->rate + mode_cost_y;
