Remove LGT_FROM_PRED experiment
This experiment has been abandonned for AV1.
Change-Id: I18cf1354df928a0614a1e58b718cd96ee7999925
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 4924f0c..804c249 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1222,7 +1222,6 @@
// is no need to send the tx_type
assert(eset > 0);
assert(av1_ext_tx_used[tx_set_type][tx_type]);
-#if !CONFIG_LGT_FROM_PRED
if (is_inter) {
aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][tx_type],
ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
@@ -1246,39 +1245,6 @@
av1_num_ext_tx_set[tx_set_type]);
#endif
}
-#else
- // only signal tx_type when lgt is not allowed or not selected
- if (is_inter) {
- if (LGT_FROM_PRED_INTER) {
- if (is_lgt_allowed(mbmi->mode, tx_size) && !cm->reduced_tx_set_used)
- aom_write(w, mbmi->use_lgt, ec_ctx->inter_lgt_prob[square_tx_size]);
- if (!mbmi->use_lgt)
- aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][tx_type],
- ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
- av1_num_ext_tx_set[tx_set_type]);
- } else {
- aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][tx_type],
- ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
- av1_num_ext_tx_set[tx_set_type]);
- }
- } else if (ALLOW_INTRA_EXT_TX) {
- if (LGT_FROM_PRED_INTRA) {
- if (is_lgt_allowed(mbmi->mode, tx_size) && !cm->reduced_tx_set_used)
- aom_write(w, mbmi->use_lgt,
- ec_ctx->intra_lgt_prob[square_tx_size][mbmi->mode]);
- if (!mbmi->use_lgt)
- aom_write_symbol(
- w, av1_ext_tx_ind[tx_set_type][tx_type],
- ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
- av1_num_ext_tx_set[tx_set_type]);
- } else {
- aom_write_symbol(
- w, av1_ext_tx_ind[tx_set_type][tx_type],
- ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
- av1_num_ext_tx_set[tx_set_type]);
- }
- }
-#endif // CONFIG_LGT_FROM_PRED
}
}
}
diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 081c9e0..04b880e 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -258,10 +258,6 @@
int quarter_tx_size_cost[2];
#endif
int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
-#if CONFIG_LGT_FROM_PRED
- int intra_lgt_cost[LGT_SIZES][INTRA_MODES][2];
- int inter_lgt_cost[LGT_SIZES][2];
-#endif
int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
[TX_TYPES];
diff --git a/av1/encoder/dct.c b/av1/encoder/dct.c
index e6bcac4..1cdeb90 100644
--- a/av1/encoder/dct.c
+++ b/av1/encoder/dct.c
@@ -1070,20 +1070,10 @@
}
#endif // CONFIG_MRC_TX
-#if CONFIG_LGT || CONFIG_LGT_FROM_PRED
+#if CONFIG_LGT
static void flgt4(const tran_low_t *input, tran_low_t *output,
const tran_high_t *lgtmtx) {
if (!lgtmtx) assert(0);
-#if CONFIG_LGT_FROM_PRED
- // For DCT/ADST, use butterfly implementations
- if (lgtmtx[0] == DCT4) {
- fdct4(input, output);
- return;
- } else if (lgtmtx[0] == ADST4) {
- fadst4(input, output);
- return;
- }
-#endif // CONFIG_LGT_FROM_PRED
// evaluate s[j] = sum of all lgtmtx[j][i]*input[i] over i=1,...,4
tran_high_t s[4] = { 0 };
@@ -1096,16 +1086,6 @@
static void flgt8(const tran_low_t *input, tran_low_t *output,
const tran_high_t *lgtmtx) {
if (!lgtmtx) assert(0);
-#if CONFIG_LGT_FROM_PRED
- // For DCT/ADST, use butterfly implementations
- if (lgtmtx[0] == DCT8) {
- fdct8(input, output);
- return;
- } else if (lgtmtx[0] == ADST8) {
- fadst8(input, output);
- return;
- }
-#endif // CONFIG_LGT_FROM_PRED
// evaluate s[j] = sum of all lgtmtx[j][i]*input[i] over i=1,...,8
tran_high_t s[8] = { 0 };
@@ -1114,140 +1094,7 @@
for (int i = 0; i < 8; ++i) output[i] = (tran_low_t)fdct_round_shift(s[i]);
}
-#endif // CONFIG_LGT || CONFIG_LGT_FROM_PRED
-
-#if CONFIG_LGT_FROM_PRED
-static void flgt16up(const tran_low_t *input, tran_low_t *output,
- const tran_high_t *lgtmtx) {
- if (lgtmtx[0] == DCT16) {
- fdct16(input, output);
- return;
- } else if (lgtmtx[0] == ADST16) {
- fadst16(input, output);
- return;
- } else if (lgtmtx[0] == DCT32) {
- fdct32(input, output);
- return;
- } else if (lgtmtx[0] == ADST32) {
- fhalfright32(input, output);
- return;
- } else {
- assert(0);
- }
-}
-
-typedef void (*FlgtFunc)(const tran_low_t *input, tran_low_t *output,
- const tran_high_t *lgtmtx);
-
-static FlgtFunc flgt_func[4] = { flgt4, flgt8, flgt16up, flgt16up };
-
-typedef void (*GetLgtFunc)(const TxfmParam *txfm_param, int is_col,
- const tran_high_t *lgtmtx[], int ntx);
-
-static GetLgtFunc get_lgt_func[4] = { get_lgt4_from_pred, get_lgt8_from_pred,
- get_lgt16up_from_pred,
- get_lgt16up_from_pred };
-
-// this inline function corresponds to the up scaling before the first
-// transform in the av1_fht* functions
-static INLINE tran_low_t fwd_upscale_wrt_txsize(const tran_high_t val,
- const TX_SIZE tx_size) {
- switch (tx_size) {
- case TX_4X4: return (tran_low_t)val << 4;
- case TX_8X8:
- case TX_4X16:
- case TX_16X4:
- case TX_8X32:
- case TX_32X8: return (tran_low_t)val << 2;
- case TX_4X8:
- case TX_8X4:
- case TX_8X16:
- case TX_16X8: return (tran_low_t)fdct_round_shift(val * 4 * Sqrt2);
- default: assert(0); break;
- }
- return 0;
-}
-
-// This inline function corresponds to the bit shift after the second
-// transform in the av1_fht* functions
-static INLINE tran_low_t fwd_downscale_wrt_txsize(const tran_low_t val,
- const TX_SIZE tx_size) {
- switch (tx_size) {
- case TX_4X4: return (val + 1) >> 2;
- case TX_4X8:
- case TX_8X4:
- case TX_8X8:
- case TX_4X16:
- case TX_16X4: return (val + (val < 0)) >> 1;
- case TX_8X16:
- case TX_16X8: return val;
- case TX_8X32:
- case TX_32X8: return ROUND_POWER_OF_TWO_SIGNED(val, 2);
- default: assert(0); break;
- }
- return 0;
-}
-
-void flgt2d_from_pred_c(const int16_t *input, tran_low_t *output, int stride,
- TxfmParam *txfm_param) {
- const TX_SIZE tx_size = txfm_param->tx_size;
- const int w = tx_size_wide[tx_size];
- const int h = tx_size_high[tx_size];
- const int wlog2 = tx_size_wide_log2[tx_size];
- const int hlog2 = tx_size_high_log2[tx_size];
- assert(w <= 8 || h <= 8);
-
- int i, j;
- tran_low_t out[256]; // max size: 8x32 and 32x8
- tran_low_t temp_in[32], temp_out[32];
- const tran_high_t *lgtmtx_col[1];
- const tran_high_t *lgtmtx_row[1];
- get_lgt_func[hlog2 - 2](txfm_param, 1, lgtmtx_col, w);
- get_lgt_func[wlog2 - 2](txfm_param, 0, lgtmtx_row, h);
-
- // For forward transforms, to be consistent with av1_fht functions, we apply
- // short transform first and long transform second.
- if (w < h) {
- // Row transforms
- for (i = 0; i < h; ++i) {
- for (j = 0; j < w; ++j)
- temp_in[j] = fwd_upscale_wrt_txsize(input[i * stride + j], tx_size);
- flgt_func[wlog2 - 2](temp_in, temp_out, lgtmtx_row[0]);
- // right shift of 2 bits here in fht8x16 and fht16x8
- for (j = 0; j < w; ++j)
- out[j * h + i] = (tx_size == TX_16X8 || tx_size == TX_8X16)
- ? ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2)
- : temp_out[j];
- }
- // Column transforms
- for (i = 0; i < w; ++i) {
- for (j = 0; j < h; ++j) temp_in[j] = out[j + i * h];
- flgt_func[hlog2 - 2](temp_in, temp_out, lgtmtx_col[0]);
- for (j = 0; j < h; ++j)
- output[j * w + i] = fwd_downscale_wrt_txsize(temp_out[j], tx_size);
- }
- } else {
- // Column transforms
- for (i = 0; i < w; ++i) {
- for (j = 0; j < h; ++j)
- temp_in[j] = fwd_upscale_wrt_txsize(input[j * stride + i], tx_size);
- flgt_func[hlog2 - 2](temp_in, temp_out, lgtmtx_col[0]);
- // fht8x16 and fht16x8 have right shift of 2 bits here
- for (j = 0; j < h; ++j)
- out[j * w + i] = (tx_size == TX_16X8 || tx_size == TX_8X16)
- ? ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2)
- : temp_out[j];
- }
- // Row transforms
- for (i = 0; i < h; ++i) {
- for (j = 0; j < w; ++j) temp_in[j] = out[j + i * w];
- flgt_func[wlog2 - 2](temp_in, temp_out, lgtmtx_row[0]);
- for (j = 0; j < w; ++j)
- output[j + i * w] = fwd_downscale_wrt_txsize(temp_out[j], tx_size);
- }
- }
-}
-#endif // CONFIG_LGT_FROM_PRED
+#endif // CONFIG_LGT
// TODO(sarahparker) these functions will be removed once the highbitdepth
// codepath works properly for rectangular transforms. They have almost
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 5090431..8a6bc09 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4485,7 +4485,6 @@
const int eset =
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (eset > 0) {
-#if !CONFIG_LGT_FROM_PRED
const TxSetType tx_set_type = get_ext_tx_set_type(
tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (is_inter) {
@@ -4525,45 +4524,6 @@
av1_num_ext_tx_set[tx_set_type]);
#endif
}
-#else
- (void)tx_type;
- (void)fc;
- (void)allow_update_cdf;
- if (is_inter) {
- if (LGT_FROM_PRED_INTER) {
- if (is_lgt_allowed(mbmi->mode, tx_size) && !cm->reduced_tx_set_used)
- ++counts->inter_lgt[txsize_sqr_map[tx_size]][mbmi->use_lgt];
-#if CONFIG_ENTROPY_STATS
- if (!mbmi->use_lgt)
- ++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type];
- else
-#endif // CONFIG_ENTROPY_STATS
- mbmi->tx_type = DCT_DCT;
- } else {
-#if CONFIG_ENTROPY_STATS
- ++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type];
-#endif // CONFIG_ENTROPY_STATS
- }
- } else {
- if (LGT_FROM_PRED_INTRA) {
- if (is_lgt_allowed(mbmi->mode, tx_size) && !cm->reduced_tx_set_used)
- ++counts->intra_lgt[txsize_sqr_map[tx_size]][mbmi->mode]
- [mbmi->use_lgt];
-#if CONFIG_ENTROPY_STATS
- if (!mbmi->use_lgt)
- ++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode]
- [tx_type];
- else
-#endif // CONFIG_ENTROPY_STATS
- mbmi->tx_type = DCT_DCT;
- } else {
-#if CONFIG_ENTROPY_STATS
- ++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode]
- [tx_type];
-#endif // CONFIG_ENTROPY_STATS
- }
- }
-#endif // CONFIG_LGT_FROM_PRED
}
}
}
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 5e3f236..1d8d9ef 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -479,7 +479,7 @@
TxfmParam txfm_param;
-#if CONFIG_DIST_8X8 || CONFIG_LGT_FROM_PRED || CONFIG_MRC_TX
+#if CONFIG_DIST_8X8 || CONFIG_MRC_TX
uint8_t *dst;
const int dst_stride = pd->dst.stride;
#if CONFIG_DIST_8X8
@@ -506,9 +506,9 @@
qparam.iqmatrix = iqmatrix;
#endif // CONFIG_AOM_QM
-#if CONFIG_DIST_8X8 || CONFIG_LGT_FROM_PRED || CONFIG_MRC_TX
+#if CONFIG_DIST_8X8 || CONFIG_MRC_TX
dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]];
-#endif // CONFIG_DIST_8X8 || CONFIG_LGT_FROM_PRED ||
+#endif // CONFIG_DIST_8X8 ||
// CONFIG_MRC_TX
#if CONFIG_DIST_8X8
@@ -545,7 +545,7 @@
#if CONFIG_MRC_TX || CONFIG_LGT
txfm_param.is_inter = is_inter_block(mbmi);
#endif
-#if CONFIG_MRC_TX || CONFIG_LGT_FROM_PRED
+#if CONFIG_MRC_TX
txfm_param.dst = dst;
txfm_param.stride = dst_stride;
#if CONFIG_MRC_TX
@@ -554,11 +554,7 @@
txfm_param.mask = BLOCK_OFFSET(xd->mrc_mask, block);
#endif // SIGNAL_ANY_MRC_MASK
#endif // CONFIG_MRC_TX
-#if CONFIG_LGT_FROM_PRED
- txfm_param.mode = mbmi->mode;
- txfm_param.use_lgt = mbmi->use_lgt;
-#endif // CONFIG_LGT_FROM_PRED
-#endif // CONFIG_MRC_TX || CONFIG_LGT_FROM_PRED
+#endif // CONFIG_MRC_TX
txfm_param.bd = xd->bd;
const int is_hbd = get_bitdepth_data_path_index(xd);
@@ -628,15 +624,9 @@
if (p->eobs[block] != 0)
{
-#if CONFIG_LGT_FROM_PRED
- PREDICTION_MODE mode = xd->mi[0]->mbmi.mode;
-#endif // CONFIG_LGT_FROM_PRED
TX_TYPE tx_type =
av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col, block, tx_size);
av1_inverse_transform_block(xd, dqcoeff,
-#if CONFIG_LGT_FROM_PRED
- mode,
-#endif
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
mrc_mask,
#endif // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
@@ -901,9 +891,6 @@
}
av1_inverse_transform_block(xd, dqcoeff,
-#if CONFIG_LGT_FROM_PRED
- xd->mi[0]->mbmi.mode,
-#endif
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
mrc_mask,
#endif // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c
index f1bed84..fdbf313 100644
--- a/av1/encoder/hybrid_fwd_txfm.c
+++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -490,14 +490,6 @@
TxfmParam *txfm_param) {
assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]);
const TX_SIZE tx_size = txfm_param->tx_size;
-#if CONFIG_LGT_FROM_PRED
- if (txfm_param->use_lgt) {
- // if use_lgt is 1, it will override tx_type
- assert(is_lgt_allowed(txfm_param->mode, tx_size));
- flgt2d_from_pred_c(src_diff, coeff, diff_stride, txfm_param);
- return;
- }
-#endif // CONFIG_LGT_FROM_PRED
switch (tx_size) {
#if CONFIG_TX64X64
case TX_64X64:
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 0390b76..877955b 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -244,22 +244,6 @@
#endif
}
-#if CONFIG_LGT_FROM_PRED
- if (LGT_FROM_PRED_INTRA) {
- for (i = 0; i < LGT_SIZES; ++i) {
- for (j = 0; j < INTRA_MODES; ++j) {
- x->intra_lgt_cost[i][j][0] = av1_cost_bit(fc->intra_lgt_prob[i][j], 0);
- x->intra_lgt_cost[i][j][1] = av1_cost_bit(fc->intra_lgt_prob[i][j], 1);
- }
- }
- }
- if (LGT_FROM_PRED_INTER) {
- for (i = 0; i < LGT_SIZES; ++i) {
- x->inter_lgt_cost[i][0] = av1_cost_bit(fc->inter_lgt_prob[i], 0);
- x->inter_lgt_cost[i][1] = av1_cost_bit(fc->inter_lgt_prob[i], 1);
- }
- }
-#endif // CONFIG_LGT_FROM_PRED
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
int s;
for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 91f4ef0..36c3aed 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2039,9 +2039,6 @@
TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, block, tx_size);
av1_inverse_transform_block(xd, dqcoeff,
-#if CONFIG_LGT_FROM_PRED
- xd->mi[0]->mbmi.mode,
-#endif
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
mrc_mask,
#endif // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
@@ -2380,38 +2377,12 @@
}
}
-#if CONFIG_LGT_FROM_PRED
-int av1_lgt_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
- const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane,
- TX_SIZE tx_size, int use_lgt) {
- if (plane > 0) return 0;
- const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
- const int is_inter = is_inter_block(mbmi);
-
- assert(is_lgt_allowed(mbmi->mode, tx_size));
- if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 &&
- !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
- const int ext_tx_set =
- get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
- if (LGT_FROM_PRED_INTRA && !is_inter && ext_tx_set > 0 &&
- ALLOW_INTRA_EXT_TX)
- return x->intra_lgt_cost[txsize_sqr_map[tx_size]][mbmi->mode][use_lgt];
- if (LGT_FROM_PRED_INTRA && is_inter && ext_tx_set > 0)
- return x->inter_lgt_cost[txsize_sqr_map[tx_size]][use_lgt];
- }
- return 0;
-}
-#endif // CONFIG_LGT_FROM_PRED
-
// TODO(angiebird): use this function whenever it's possible
int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane,
TX_SIZE tx_size, TX_TYPE tx_type) {
if (plane > 0) return 0;
-#if CONFIG_LGT_FROM_PRED
- assert(!xd->mi[0]->mbmi.use_lgt);
-#endif
tx_size = get_min_tx_size(tx_size);
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
@@ -2477,15 +2448,7 @@
if (rd_stats->rate == INT_MAX) return INT64_MAX;
#if !CONFIG_TXK_SEL
int plane = 0;
-#if CONFIG_LGT_FROM_PRED
- if (is_lgt_allowed(mbmi->mode, tx_size))
- rd_stats->rate +=
- av1_lgt_cost(cm, x, xd, bs, plane, tx_size, mbmi->use_lgt);
- if (!mbmi->use_lgt)
- rd_stats->rate += av1_tx_type_cost(cm, x, xd, bs, plane, tx_size, tx_type);
-#else
rd_stats->rate += av1_tx_type_cost(cm, x, xd, bs, plane, tx_size, tx_type);
-#endif // CONFIG_LGT_FROM_PRED
#endif
if (rd_stats->skip) {
@@ -2523,9 +2486,6 @@
tx_size != TX_32X32))
return 1;
#endif // CONFIG_MRC_TX
-#if CONFIG_LGT_FROM_PRED
- if (mbmi->use_lgt && mbmi->ref_mv_idx > 0) return 1;
-#endif // CONFIG_LGT_FROM_PRED
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) return 1;
if (FIXED_TX_TYPE && tx_type != get_default_tx_type(0, xd, 0, tx_size))
return 1;
@@ -2580,14 +2540,6 @@
const int is_inter = is_inter_block(mbmi);
int prune = 0;
const int plane = 0;
-#if CONFIG_LGT_FROM_PRED
- int is_lgt_best = 0;
- int search_lgt = is_inter
- ? LGT_FROM_PRED_INTER && !x->use_default_inter_tx_type &&
- !cpi->sf.tx_type_search.prune_mode > NO_PRUNE
- : LGT_FROM_PRED_INTRA && !x->use_default_intra_tx_type &&
- ALLOW_INTRA_EXT_TX;
-#endif // CONFIG_LGT_FROM_PRED
av1_invalid_rd_stats(rd_stats);
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
@@ -2647,42 +2599,12 @@
}
}
-#if CONFIG_LGT_FROM_PRED
- // search LGT
- if (search_lgt && is_lgt_allowed(mbmi->mode, mbmi->tx_size) &&
- !cm->reduced_tx_set_used) {
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- txfm_rd_in_plane(x, cpi, &this_rd_stats, ref_best_rd, 0, bs,
- mbmi->tx_size, cpi->sf.use_fast_coef_costing);
- if (this_rd_stats.rate != INT_MAX) {
- av1_lgt_cost(cm, x, xd, bs, plane, mbmi->tx_size, 1);
- if (this_rd_stats.skip)
- this_rd = RDCOST(x->rdmult, s1, this_rd_stats.sse);
- else
- this_rd =
- RDCOST(x->rdmult, this_rd_stats.rate + s0, this_rd_stats.dist);
- if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] &&
- !this_rd_stats.skip)
- this_rd = AOMMIN(this_rd, RDCOST(x->rdmult, s1, this_rd_stats.sse));
- if (this_rd < best_rd) {
- best_rd = this_rd;
- is_lgt_best = 1;
- *rd_stats = this_rd_stats;
- }
- }
- mbmi->use_lgt = 0;
- }
-#endif // CONFIG_LGT_FROM_PRED
} else {
mbmi->tx_type = DCT_DCT;
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size,
cpi->sf.use_fast_coef_costing);
}
mbmi->tx_type = best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = is_lgt_best;
-#endif // CONFIG_LGT_FROM_PRED
}
static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
@@ -2717,11 +2639,6 @@
const TX_SIZE max_tx_size = max_txsize_lookup[bs];
TX_SIZE best_tx_size = max_tx_size;
TX_TYPE best_tx_type = DCT_DCT;
-#if CONFIG_LGT_FROM_PRED
- int breakout = 0;
- int is_lgt_best = 0;
- mbmi->use_lgt = 0;
-#endif // CONFIG_LGT_FROM_PRED
#if CONFIG_TXK_SEL
TX_TYPE best_txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif // CONFIG_TXK_SEL
@@ -2773,21 +2690,6 @@
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
-#if CONFIG_LGT_FROM_PRED
- const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
- if (is_lgt_allowed(mbmi->mode, rect_tx_size) && !cm->reduced_tx_set_used) {
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, 0, rect_tx_size);
- if (rd < best_rd) {
- is_lgt_best = 1;
- best_tx_size = rect_tx_size;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- mbmi->use_lgt = 0;
- }
-#endif // CONFIG_LGT_FROM_PRED
}
#if CONFIG_RECT_TX_EXT
@@ -2826,9 +2728,6 @@
sizeof(best_txk_type[0]) * num_blk);
#endif
best_tx_type = tx_type;
-#if CONFIG_LGT_FROM_PRED
- is_lgt_best = 0;
-#endif
best_tx_size = tx_size;
best_rd = rd;
*rd_stats = this_rd_stats;
@@ -2839,21 +2738,6 @@
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
-#if CONFIG_LGT_FROM_PRED
- if (is_lgt_allowed(mbmi->mode, tx_size) && !cm->reduced_tx_set_used) {
- const TX_SIZE tx_size = quarter_txsize_lookup[bs];
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, 0, tx_size);
- if (rd < best_rd) {
- is_lgt_best = 1;
- best_tx_size = tx_size;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- mbmi->use_lgt = 0;
- }
-#endif // CONFIG_LGT_FROM_PRED
}
#endif // CONFIG_RECT_TX_EXT
@@ -2894,9 +2778,6 @@
(rd == INT64_MAX ||
(this_rd_stats.skip == 1 && tx_type != DCT_DCT && n < start_tx) ||
(n < (int)max_tx_size && rd > last_rd))) {
-#if CONFIG_LGT_FROM_PRED
- breakout = 1;
-#endif
break;
}
@@ -2907,9 +2788,6 @@
memcpy(best_txk_type, mbmi->txk_type, sizeof(best_txk_type[0]) * 256);
#endif
best_tx_type = tx_type;
-#if CONFIG_LGT_FROM_PRED
- is_lgt_best = 0;
-#endif
best_tx_size = n;
best_rd = rd;
*rd_stats = this_rd_stats;
@@ -2919,28 +2797,9 @@
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 1;
- if (is_lgt_allowed(mbmi->mode, n) &&
- !skip_txfm_search(cpi, x, bs, 0, n, prune) && !breakout) {
- RD_STATS this_rd_stats;
- rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, 0, n);
- if (rd < best_rd) {
- is_lgt_best = 1;
- best_tx_size = n;
- best_rd = rd;
- *rd_stats = this_rd_stats;
- }
- }
- mbmi->use_lgt = 0;
-#endif // CONFIG_LGT_FROM_PRED
}
mbmi->tx_size = best_tx_size;
mbmi->tx_type = best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = is_lgt_best;
- assert(!is_lgt_best || is_lgt_allowed(mbmi->mode, mbmi->tx_size));
-#endif // CONFIG_LGT_FROM_PRED
#if CONFIG_TXK_SEL
memcpy(mbmi->txk_type, best_txk_type, sizeof(best_txk_type[0]) * 256);
#endif
@@ -3278,9 +3137,6 @@
TX_SIZE best_tx_size = TX_4X4;
FILTER_INTRA_MODE_INFO filter_intra_mode_info;
TX_TYPE best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- int use_lgt_when_selected;
-#endif
av1_zero(filter_intra_mode_info);
mbmi->filter_intra_mode_info.use_filter_intra_mode[0] = 1;
@@ -3310,9 +3166,6 @@
best_tx_size = mic->mbmi.tx_size;
filter_intra_mode_info = mbmi->filter_intra_mode_info;
best_tx_type = mic->mbmi.tx_type;
-#if CONFIG_LGT_FROM_PRED
- use_lgt_when_selected = mic->mbmi.use_lgt;
-#endif
*rate = this_rate;
*rate_tokenonly = tokenonly_rd_stats.rate;
*distortion = tokenonly_rd_stats.dist;
@@ -3324,9 +3177,6 @@
if (filter_intra_selected_flag) {
mbmi->mode = DC_PRED;
mbmi->tx_size = best_tx_size;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = use_lgt_when_selected;
-#endif
mbmi->filter_intra_mode_info.use_filter_intra_mode[0] =
filter_intra_mode_info.use_filter_intra_mode[0];
mbmi->filter_intra_mode_info.filter_intra_mode[0] =
@@ -3346,11 +3196,7 @@
const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize, int mode_cost,
int64_t best_rd_in, int8_t angle_delta, int max_angle_delta, int *rate,
RD_STATS *rd_stats, int *best_angle_delta, TX_SIZE *best_tx_size,
- TX_TYPE *best_tx_type,
-#if CONFIG_LGT_FROM_PRED
- int *use_lgt_when_selected,
-#endif
- int64_t *best_rd, int64_t *best_model_rd) {
+ TX_TYPE *best_tx_type, int64_t *best_rd, int64_t *best_model_rd) {
int this_rate;
RD_STATS tokenonly_rd_stats;
int64_t this_rd, this_model_rd;
@@ -3381,9 +3227,6 @@
*best_angle_delta = mbmi->angle_delta[0];
*best_tx_size = mbmi->tx_size;
*best_tx_type = mbmi->tx_type;
-#if CONFIG_LGT_FROM_PRED
- *use_lgt_when_selected = mbmi->use_lgt;
-#endif
*rate = this_rate;
rd_stats->rate = tokenonly_rd_stats.rate;
rd_stats->dist = tokenonly_rd_stats.dist;
@@ -3408,9 +3251,6 @@
int64_t this_rd, best_rd_in, rd_cost[2 * (MAX_ANGLE_DELTA + 2)];
TX_SIZE best_tx_size = mic->mbmi.tx_size;
TX_TYPE best_tx_type = mbmi->tx_type;
-#if CONFIG_LGT_FROM_PRED
- int use_lgt_when_selected = mbmi->use_lgt;
-#endif
for (i = 0; i < 2 * (MAX_ANGLE_DELTA + 2); ++i) rd_cost[i] = INT64_MAX;
@@ -3422,11 +3262,7 @@
this_rd = calc_rd_given_intra_angle(
cpi, x, bsize, mode_cost, best_rd_in, (1 - 2 * i) * angle_delta,
MAX_ANGLE_DELTA, rate, rd_stats, &best_angle_delta, &best_tx_size,
- &best_tx_type,
-#if CONFIG_LGT_FROM_PRED
- &use_lgt_when_selected,
-#endif
- &best_rd, best_model_rd);
+ &best_tx_type, &best_rd, best_model_rd);
rd_cost[2 * angle_delta + i] = this_rd;
if (first_try && this_rd == INT64_MAX) return best_rd;
first_try = 0;
@@ -3447,14 +3283,10 @@
rd_cost[2 * (angle_delta - 1) + i] > rd_thresh)
skip_search = 1;
if (!skip_search) {
- calc_rd_given_intra_angle(cpi, x, bsize, mode_cost, best_rd,
- (1 - 2 * i) * angle_delta, MAX_ANGLE_DELTA,
- rate, rd_stats, &best_angle_delta,
- &best_tx_size, &best_tx_type,
-#if CONFIG_LGT_FROM_PRED
- &use_lgt_when_selected,
-#endif
- &best_rd, best_model_rd);
+ calc_rd_given_intra_angle(
+ cpi, x, bsize, mode_cost, best_rd, (1 - 2 * i) * angle_delta,
+ MAX_ANGLE_DELTA, rate, rd_stats, &best_angle_delta, &best_tx_size,
+ &best_tx_type, &best_rd, best_model_rd);
}
}
}
@@ -3462,9 +3294,6 @@
mbmi->tx_size = best_tx_size;
mbmi->angle_delta[0] = best_angle_delta;
mbmi->tx_type = best_tx_type;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = use_lgt_when_selected;
-#endif
return best_rd;
}
@@ -3993,9 +3822,6 @@
const int eob = p->eobs[block];
av1_inverse_transform_block(xd, dqcoeff,
-#if CONFIG_LGT_FROM_PRED
- xd->mi[0]->mbmi.mode,
-#endif
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
mrc_mask,
#endif // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
@@ -4603,46 +4429,30 @@
!xd->lossless[xd->mi[0]->mbmi.segment_id]) {
const int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter,
cm->reduced_tx_set_used);
-#if CONFIG_LGT_FROM_PRED
- if (is_lgt_allowed(mbmi->mode, mbmi->min_tx_size)) {
- if (LGT_FROM_PRED_INTRA && !is_inter && ext_tx_set > 0 &&
- ALLOW_INTRA_EXT_TX)
- rd_stats->rate += x->intra_lgt_cost[txsize_sqr_map[mbmi->min_tx_size]]
- [mbmi->mode][mbmi->use_lgt];
- if (LGT_FROM_PRED_INTER && is_inter && ext_tx_set > 0)
+ if (is_inter) {
+ if (ext_tx_set > 0)
rd_stats->rate +=
- x->inter_lgt_cost[txsize_sqr_map[mbmi->min_tx_size]][mbmi->use_lgt];
- }
- if (!mbmi->use_lgt) {
-#endif // CONFIG_LGT_FROM_PRED
- if (is_inter) {
- if (ext_tx_set > 0)
- rd_stats->rate +=
- x->inter_tx_type_costs[ext_tx_set]
- [txsize_sqr_map[mbmi->min_tx_size]]
- [mbmi->tx_type];
- } else {
- if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
+ x->inter_tx_type_costs[ext_tx_set]
+ [txsize_sqr_map[mbmi->min_tx_size]]
+ [mbmi->tx_type];
+ } else {
+ if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
#if CONFIG_FILTER_INTRA
- PREDICTION_MODE intra_dir;
- if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0])
- intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
- .filter_intra_mode[0]];
- else
- intra_dir = mbmi->mode;
- rd_stats->rate +=
- x->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size][intra_dir]
- [mbmi->tx_type];
+ PREDICTION_MODE intra_dir;
+ if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0])
+ intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
+ .filter_intra_mode[0]];
+ else
+ intra_dir = mbmi->mode;
+ rd_stats->rate += x->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size]
+ [intra_dir][mbmi->tx_type];
#else
rd_stats->rate += x->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size]
[mbmi->mode][mbmi->tx_type];
#endif
- }
}
}
-#if CONFIG_LGT_FROM_PRED
}
-#endif
#endif // CONFIG_TXK_SEL
if (rd_stats->skip)
@@ -5022,14 +4832,6 @@
av1_invalid_rd_stats(rd_stats);
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
- int search_lgt = is_inter
- ? LGT_FROM_PRED_INTER &&
- (!cpi->sf.tx_type_search.prune_mode > NO_PRUNE)
- : LGT_FROM_PRED_INTRA && ALLOW_INTRA_EXT_TX;
-#endif // CONFIG_LGT_FROM_PRED
-
const uint32_t hash = get_block_residue_hash(x, bsize);
TX_RD_RECORD *tx_rd_record = &x->tx_rd_record;
@@ -5132,27 +4934,6 @@
assert(IMPLIES(!found, ref_best_rd != INT64_MAX));
if (!found) return;
-#if CONFIG_LGT_FROM_PRED
- if (search_lgt && is_lgt_allowed(mbmi->mode, max_tx_size) &&
- !cm->reduced_tx_set_used) {
- RD_STATS this_rd_stats;
- mbmi->use_lgt = 1;
- rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, mi_row, mi_col,
- ref_best_rd, 0, 0);
- if (rd < best_rd) {
- best_rd = rd;
- *rd_stats = this_rd_stats;
- best_tx = mbmi->tx_size;
- best_min_tx_size = mbmi->min_tx_size;
- memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4);
- for (idy = 0; idy < xd->n8_h; ++idy)
- for (idx = 0; idx < xd->n8_w; ++idx)
- best_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx];
- } else {
- mbmi->use_lgt = 0;
- }
- }
-#endif // CONFIG_LGT_FROM_PRED
// We found a candidate transform to use. Copy our results from the "best"
// array into mbmi.
mbmi->tx_type = best_tx_type;
@@ -8447,9 +8228,6 @@
mbmi->use_wedge_interintra = 0;
int compmode_interinter_cost = 0;
mbmi->interinter_compound_type = COMPOUND_AVERAGE;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
-#endif
if (!cm->allow_interintra_compound && is_comp_interintra_pred)
return INT64_MAX;
@@ -9224,9 +9002,6 @@
mbmi->use_intrabc = 0;
mbmi->mv[0].as_int = 0;
#endif // CONFIG_INTRABC
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
-#endif
const int64_t intra_yrd = rd_pick_intra_sby_mode(
cpi, x, &rate_y, &rate_y_tokenonly, &dist_y, &y_skip, bsize, best_rd);
@@ -10863,9 +10638,6 @@
int idx, idy;
best_mbmode.tx_type = mbmi->tx_type;
best_mbmode.tx_size = mbmi->tx_size;
-#if CONFIG_LGT_FROM_PRED
- best_mbmode.use_lgt = mbmi->use_lgt;
-#endif
for (idy = 0; idy < xd->n8_h; ++idy)
for (idx = 0; idx < xd->n8_w; ++idx)
best_mbmode.inter_tx_size[idy][idx] = mbmi->inter_tx_size[idy][idx];
@@ -11225,9 +10997,6 @@
mbmi->ref_mv_idx = 0;
mbmi->pred_mv[0].as_int = 0;
-#if CONFIG_LGT_FROM_PRED
- mbmi->use_lgt = 0;
-#endif
mbmi->motion_mode = SIMPLE_TRANSLATION;
av1_count_overlappable_neighbors(cm, xd, mi_row, mi_col);
