Remove LGT_FROM_PRED experiment
This experiment has been abandonned for AV1.
Change-Id: I18cf1354df928a0614a1e58b718cd96ee7999925
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 91f4ef0..36c3aed 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2039,9 +2039,6 @@
TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, block, tx_size);
av1_inverse_transform_block(xd, dqcoeff,
-#if CONFIG_LGT_FROM_PRED
- xd->mi[0]->mbmi.mode,
-#endif
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
mrc_mask,
#endif // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
@@ -2380,38 +2377,12 @@
}
}
-#if CONFIG_LGT_FROM_PRED
-int av1_lgt_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
- const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane,
- TX_SIZE tx_size, int use_lgt) {
- if (plane > 0) return 0;
- const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
- const int is_inter = is_inter_block(mbmi);
-
- assert(is_lgt_allowed(mbmi->mode, tx_size));
- if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 &&
- !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
- const int ext_tx_set =
- get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
- if (LGT_FROM_PRED_INTRA && !is_inter && ext_tx_set > 0 &&
- ALLOW_INTRA_EXT_TX)
- return x->intra_lgt_cost[txsize_sqr_map[tx_size]][mbmi->mode][use_lgt];
- if (LGT_FROM_PRED_INTRA && is_inter && ext_tx_set > 0)
- return x->inter_lgt_cost[txsize_sqr_map[tx_size]][use_lgt];
- }
- return 0;
-}
-#endif // CONFIG_LGT_FROM_PRED
-
// TODO(angiebird): use this function whenever it's possible
int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane,
TX_SIZE tx_size, TX_TYPE tx_type) {
if (plane > 0) return 0;
-#if CONFIG_LGT_FROM_PRED
- assert(!xd->mi[0]->mbmi.use_lgt);
-#endif
tx_size = get_min_tx_size(tx_size);
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
@@ -2477,15 +2448,7 @@
if (rd_stats->rate == INT_MAX) return INT64_MAX;
#if !CONFIG_TXK_SEL
int plane = 0;
-#if CONFIG_LGT_FROM_PRED
- if (is_lgt_allowed(mbmi->mode, tx_size))
- rd_stats->rate +=
- av1_lgt_cost(cm, x, xd, bs, plane, tx_size, mbmi->use_lgt);
- if (!mbmi->use_lgt)
- rd_stats->rate += av1_tx_type_cost(cm, x, xd, bs, plane, tx_size, tx_type);
-#else
rd_stats->rate += av1_tx_type_cost(cm, x, xd, bs, plane, tx_size, tx_type);
-#endif // CONFIG_LGT_FROM_PRED
#endif
if (rd_stats->skip) {
@@ -2523,9 +2486,6 @@
tx_size != TX_32X32))
return 1;
#endif // CONFIG_MRC_TX
-#if CONFIG_LGT_FROM_PRED
- if (mbmi->use_lgt && mbmi->ref_mv_idx > 0) return 1;
-#endif // CONFIG_LGT_FROM_PRED
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) return 1;
if (FIXED_TX_TYPE && tx_type != get_default_tx_type(0, xd, 0, tx_size))
return 1;
@@ -2580,14 +2540,6 @@
const int is_inter = is_inter_block(mbmi);
int prune = 0;
const int plane = 0;
-#if CONFIG_LGT_FROM_PRED
- int is_lgt_best = 0;
- int search_lgt = is_inter
- ? LGT_FROM_PRED_INTER && !x->use_default_inter_tx_type &&
- !cpi->sf.tx_type_search.prune_mode > NO_PRUNE
- : LGT_FROM_PRED_INTRA && !x->use_default_intra_tx_type &&
- ALLOW_INTRA_EXT_TX;
-#endif // CONFIG_LGT_FROM_PRED
av1_invalid_rd_stats(rd_stats);
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
@@ -2647,42 +2599,12 @@
}
}
-#if CONFIG_LGT_FROM_PRED
- // search LGT
- if (search_lgt && is_lgt_allowed(mbmi->mode, mbmi->tx_size) &&
- !cm->reduced_tx_set_used) {
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- txfm_rd_in_plane(x, cpi, &this_rd_stats, ref_best_rd, 0, bs,
- mbmi->tx_size, cpi->sf.use_fast_coef_costing);
- if (this_rd_stats.rate != INT_MAX) {
- av1_lgt_cost(cm, x, xd, bs, plane, mbmi->tx_size, 1);
- if (this_rd_stats.skip)
- this_rd = RDCOST(x->rdmult, s1, this_rd_stats.sse);
- else
- this_rd =
- RDCOST(x->rdmult, this_rd_stats.rate + s0, this_rd_stats.dist);
- if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] &&
- !this_rd_stats.skip)
- this_rd = AOMMIN(this_rd, RDCOST(x->rdmult, s1, this_rd_stats.sse));
- if (this_rd < best_rd) {
- best_rd = this_rd;
- is_lgt_best = 1;
- *rd_stats = this_rd_stats;
- }
- }
- mbmi->use_lgt = 0;
- }
-#endif // CONFIG_LGT_FROM_PRED
} else {
mbmi->tx_type = DCT_DCT;
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size,
cpi->sf.use_fast_coef_costing);
}
mbmi->tx_type = best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = is_lgt_best;
-#endif // CONFIG_LGT_FROM_PRED
}
static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
@@ -2717,11 +2639,6 @@
const TX_SIZE max_tx_size = max_txsize_lookup[bs];
TX_SIZE best_tx_size = max_tx_size;
TX_TYPE best_tx_type = DCT_DCT;
-#if CONFIG_LGT_FROM_PRED
- int breakout = 0;
- int is_lgt_best = 0;
- mbmi->use_lgt = 0;
-#endif // CONFIG_LGT_FROM_PRED
#if CONFIG_TXK_SEL
TX_TYPE best_txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif // CONFIG_TXK_SEL
@@ -2773,21 +2690,6 @@
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
-#if CONFIG_LGT_FROM_PRED
- const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
- if (is_lgt_allowed(mbmi->mode, rect_tx_size) && !cm->reduced_tx_set_used) {
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, 0, rect_tx_size);
- if (rd < best_rd) {
- is_lgt_best = 1;
- best_tx_size = rect_tx_size;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- mbmi->use_lgt = 0;
- }
-#endif // CONFIG_LGT_FROM_PRED
}
#if CONFIG_RECT_TX_EXT
@@ -2826,9 +2728,6 @@
sizeof(best_txk_type[0]) * num_blk);
#endif
best_tx_type = tx_type;
-#if CONFIG_LGT_FROM_PRED
- is_lgt_best = 0;
-#endif
best_tx_size = tx_size;
best_rd = rd;
*rd_stats = this_rd_stats;
@@ -2839,21 +2738,6 @@
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
-#if CONFIG_LGT_FROM_PRED
- if (is_lgt_allowed(mbmi->mode, tx_size) && !cm->reduced_tx_set_used) {
- const TX_SIZE tx_size = quarter_txsize_lookup[bs];
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, 0, tx_size);
- if (rd < best_rd) {
- is_lgt_best = 1;
- best_tx_size = tx_size;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- mbmi->use_lgt = 0;
- }
-#endif // CONFIG_LGT_FROM_PRED
}
#endif // CONFIG_RECT_TX_EXT
@@ -2894,9 +2778,6 @@
(rd == INT64_MAX ||
(this_rd_stats.skip == 1 && tx_type != DCT_DCT && n < start_tx) ||
(n < (int)max_tx_size && rd > last_rd))) {
-#if CONFIG_LGT_FROM_PRED
- breakout = 1;
-#endif
break;
}
@@ -2907,9 +2788,6 @@
memcpy(best_txk_type, mbmi->txk_type, sizeof(best_txk_type[0]) * 256);
#endif
best_tx_type = tx_type;
-#if CONFIG_LGT_FROM_PRED
- is_lgt_best = 0;
-#endif
best_tx_size = n;
best_rd = rd;
*rd_stats = this_rd_stats;
@@ -2919,28 +2797,9 @@
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 1;
- if (is_lgt_allowed(mbmi->mode, n) &&
- !skip_txfm_search(cpi, x, bs, 0, n, prune) && !breakout) {
- RD_STATS this_rd_stats;
- rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, 0, n);
- if (rd < best_rd) {
- is_lgt_best = 1;
- best_tx_size = n;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- }
- mbmi->use_lgt = 0;
-#endif // CONFIG_LGT_FROM_PRED
}
mbmi->tx_size = best_tx_size;
mbmi->tx_type = best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = is_lgt_best;
- assert(!is_lgt_best || is_lgt_allowed(mbmi->mode, mbmi->tx_size));
-#endif // CONFIG_LGT_FROM_PRED
#if CONFIG_TXK_SEL
memcpy(mbmi->txk_type, best_txk_type, sizeof(best_txk_type[0]) * 256);
#endif
@@ -3278,9 +3137,6 @@
TX_SIZE best_tx_size = TX_4X4;
FILTER_INTRA_MODE_INFO filter_intra_mode_info;
TX_TYPE best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- int use_lgt_when_selected;
-#endif
av1_zero(filter_intra_mode_info);
mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 1;
@@ -3310,9 +3166,6 @@
best_tx_size = mic->mbmi.tx_size;
filter_intra_mode_info = mbmi->filter_intra_mode_info;
best_tx_type = mic->mbmi.tx_type;
-#if CONFIG_LGT_FROM_PRED
- use_lgt_when_selected = mic->mbmi.use_lgt;
-#endif
*rate = this_rate;
*rate_tokenonly = tokenonly_rd_stats.rate;
*distortion = tokenonly_rd_stats.dist;
@@ -3324,9 +3177,6 @@
if (filter_intra_selected_flag) {
mbmi->mode = DC_PRED;
mbmi->tx_size = best_tx_size;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = use_lgt_when_selected;
-#endif
mbmi->filter_intra_mode_info.use_filter_intra_mode[0] =
filter_intra_mode_info.use_filter_intra_mode[0];
mbmi->filter_intra_mode_info.filter_intra_mode[0] =
@@ -3346,11 +3196,7 @@
const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mode_cost,
int64_t best_rd_in, int8_t angle_delta, int max_angle_delta, int *rate,
RD_STATS *rd_stats, int *best_angle_delta, TX_SIZE *best_tx_size,
- TX_TYPE *best_tx_type,
-#if CONFIG_LGT_FROM_PRED
- int *use_lgt_when_selected,
-#endif
- int64_t *best_rd, int64_t *best_model_rd) {
+ TX_TYPE *best_tx_type, int64_t *best_rd, int64_t *best_model_rd) {
int this_rate;
RD_STATS tokenonly_rd_stats;
int64_t this_rd, this_model_rd;
@@ -3381,9 +3227,6 @@
*best_angle_delta = mbmi->angle_delta[0];
*best_tx_size = mbmi->tx_size;
*best_tx_type = mbmi->tx_type;
-#if CONFIG_LGT_FROM_PRED
- *use_lgt_when_selected = mbmi->use_lgt;
-#endif
*rate = this_rate;
rd_stats->rate = tokenonly_rd_stats.rate;
rd_stats->dist = tokenonly_rd_stats.dist;
@@ -3408,9 +3251,6 @@
int64_t this_rd, best_rd_in, rd_cost[2 * (MAX_ANGLE_DELTA + 2)];
TX_SIZE best_tx_size = mic->mbmi.tx_size;
TX_TYPE best_tx_type = mbmi->tx_type;
-#if CONFIG_LGT_FROM_PRED
- int use_lgt_when_selected = mbmi->use_lgt;
-#endif
for (i = 0; i < 2 * (MAX_ANGLE_DELTA + 2); ++i) rd_cost[i] = INT64_MAX;
@@ -3422,11 +3262,7 @@
this_rd = calc_rd_given_intra_angle(
cpi, x, bsize, mode_cost, best_rd_in, (1 - 2 * i) * angle_delta,
MAX_ANGLE_DELTA, rate, rd_stats, &best_angle_delta, &best_tx_size,
- &best_tx_type,
-#if CONFIG_LGT_FROM_PRED
- &use_lgt_when_selected,
-#endif
- &best_rd, best_model_rd);
+ &best_tx_type, &best_rd, best_model_rd);
rd_cost[2 * angle_delta + i] = this_rd;
if (first_try && this_rd == INT64_MAX) return best_rd;
first_try = 0;
@@ -3447,14 +3283,10 @@
rd_cost[2 * (angle_delta - 1) + i] > rd_thresh)
skip_search = 1;
if (!skip_search) {
- calc_rd_given_intra_angle(cpi, x, bsize, mode_cost, best_rd,
- (1 - 2 * i) * angle_delta, MAX_ANGLE_DELTA,
- rate, rd_stats, &best_angle_delta,
- &best_tx_size, &best_tx_type,
-#if CONFIG_LGT_FROM_PRED
- &use_lgt_when_selected,
-#endif
- &best_rd, best_model_rd);
+ calc_rd_given_intra_angle(
+ cpi, x, bsize, mode_cost, best_rd, (1 - 2 * i) * angle_delta,
+ MAX_ANGLE_DELTA, rate, rd_stats, &best_angle_delta, &best_tx_size,
+ &best_tx_type, &best_rd, best_model_rd);
}
}
}
@@ -3462,9 +3294,6 @@
mbmi->tx_size = best_tx_size;
mbmi->angle_delta[0] = best_angle_delta;
mbmi->tx_type = best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = use_lgt_when_selected;
-#endif
return best_rd;
}
@@ -3993,9 +3822,6 @@
const int eob = p->eobs[block];
av1_inverse_transform_block(xd, dqcoeff,
-#if CONFIG_LGT_FROM_PRED
- xd->mi[0]->mbmi.mode,
-#endif
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
mrc_mask,
#endif // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
@@ -4603,46 +4429,30 @@
!xd->lossless[xd->mi[0]->mbmi.segment_id]) {
const int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter,
cm->reduced_tx_set_used);
-#if CONFIG_LGT_FROM_PRED
- if (is_lgt_allowed(mbmi->mode, mbmi->min_tx_size)) {
- if (LGT_FROM_PRED_INTRA && !is_inter && ext_tx_set > 0 &&
- ALLOW_INTRA_EXT_TX)
- rd_stats->rate += x->intra_lgt_cost[txsize_sqr_map[mbmi->min_tx_size]]
- [mbmi->mode][mbmi->use_lgt];
- if (LGT_FROM_PRED_INTER && is_inter && ext_tx_set > 0)
+ if (is_inter) {
+ if (ext_tx_set > 0)
rd_stats->rate +=
- x->inter_lgt_cost[txsize_sqr_map[mbmi->min_tx_size]][mbmi->use_lgt];
- }
- if (!mbmi->use_lgt) {
-#endif // CONFIG_LGT_FROM_PRED
- if (is_inter) {
- if (ext_tx_set > 0)
- rd_stats->rate +=
- x->inter_tx_type_costs[ext_tx_set]
- [txsize_sqr_map[mbmi->min_tx_size]]
- [mbmi->tx_type];
- } else {
- if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
+ x->inter_tx_type_costs[ext_tx_set]
+ [txsize_sqr_map[mbmi->min_tx_size]]
+ [mbmi->tx_type];
+ } else {
+ if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
#if CONFIG_FILTER_INTRA
- PREDICTION_MODE intra_dir;
- if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0])
- intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
- .filter_intra_mode[0]];
- else
- intra_dir = mbmi->mode;
- rd_stats->rate +=
- x->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size][intra_dir]
- [mbmi->tx_type];
+ PREDICTION_MODE intra_dir;
+ if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0])
+ intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
+ .filter_intra_mode[0]];
+ else
+ intra_dir = mbmi->mode;
+ rd_stats->rate += x->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size]
+ [intra_dir][mbmi->tx_type];
#else
rd_stats->rate += x->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size]
[mbmi->mode][mbmi->tx_type];
#endif
- }
}
}
-#if CONFIG_LGT_FROM_PRED
}
-#endif
#endif // CONFIG_TXK_SEL
if (rd_stats->skip)
@@ -5022,14 +4832,6 @@
av1_invalid_rd_stats(rd_stats);
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
- int search_lgt = is_inter
- ? LGT_FROM_PRED_INTER &&
- (!cpi->sf.tx_type_search.prune_mode > NO_PRUNE)
- : LGT_FROM_PRED_INTRA && ALLOW_INTRA_EXT_TX;
-#endif // CONFIG_LGT_FROM_PRED
-
const uint32_t hash = get_block_residue_hash(x, bsize);
TX_RD_RECORD *tx_rd_record = &x->tx_rd_record;
@@ -5132,27 +4934,6 @@
assert(IMPLIES(!found, ref_best_rd != INT64_MAX));
if (!found) return;
-#if CONFIG_LGT_FROM_PRED
- if (search_lgt && is_lgt_allowed(mbmi->mode, max_tx_size) &&
- !cm->reduced_tx_set_used) {
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, mi_row, mi_col,
- ref_best_rd, 0, 0);
- if (rd < best_rd) {
- best_rd = rd;
- *rd_stats = this_rd_stats;
- best_tx = mbmi->tx_size;
- best_min_tx_size = mbmi->min_tx_size;
- memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4);
- for (idy = 0; idy < xd->n8_h; ++idy)
- for (idx = 0; idx < xd->n8_w; ++idx)
- best_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx];
- } else {
- mbmi->use_lgt = 0;
- }
- }
-#endif // CONFIG_LGT_FROM_PRED
// We found a candidate transform to use. Copy our results from the "best"
// array into mbmi.
mbmi->tx_type = best_tx_type;
@@ -8447,9 +8228,6 @@
mbmi->use_wedge_interintra = 0;
int compmode_interinter_cost = 0;
mbmi->interinter_compound_type = COMPOUND_AVERAGE;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
-#endif
if (!cm->allow_interintra_compound && is_comp_interintra_pred)
return INT64_MAX;
@@ -9224,9 +9002,6 @@
mbmi->use_intrabc = 0;
mbmi->mv[0].as_int = 0;
#endif // CONFIG_INTRABC
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
-#endif
const int64_t intra_yrd = rd_pick_intra_sby_mode(
cpi, x, &rate_y, &rate_y_tokenonly, &dist_y, &y_skip, bsize, best_rd);
@@ -10863,9 +10638,6 @@
int idx, idy;
best_mbmode.tx_type = mbmi->tx_type;
best_mbmode.tx_size = mbmi->tx_size;
-#if CONFIG_LGT_FROM_PRED
- best_mbmode.use_lgt = mbmi->use_lgt;
-#endif
for (idy = 0; idy < xd->n8_h; ++idy)
for (idx = 0; idx < xd->n8_w; ++idx)
best_mbmode.inter_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx];
@@ -11225,9 +10997,6 @@
mbmi->ref_mv_idx = 0;
mbmi->pred_mv[0].as_int = 0;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
-#endif
mbmi->motion_mode = SIMPLE_TRANSLATION;
av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col);
