lgt-from-pred: transforms based on prediction
In this experiment, sharp image discontinuity in the predicted
block is detected. Based on this discontinuity, we choose
particular LGTs as row and column transforms.
Bitstream syntax, entropy coding, and RD search for LGT are added.
One binary symbol is used to signal whether LGT is used. This
experiment can work independently with the lgt experiment.
lowres: -0.414% for key frames, -0.151% overall
midres: -0.413% for key frames, -0.161% overall
Change-Id: Iaa2f2c2839c34ca4134fa55e77870dc3f1fa879f
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 600d693..207f1e2 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -2653,6 +2653,23 @@
};
#endif
+#if CONFIG_LGT_FROM_PRED
+static const aom_prob default_intra_lgt_prob[LGT_SIZES][INTRA_MODES] = {
+ { 255, 208, 208, 180, 230, 208, 194, 214, 220, 255,
+#if CONFIG_SMOOTH_HV
+ 220, 220,
+#endif
+ 230 },
+ { 255, 192, 216, 180, 180, 180, 180, 200, 200, 255,
+#if CONFIG_SMOOTH_HV
+ 220, 220,
+#endif
+ 222 },
+};
+
+static const aom_prob default_inter_lgt_prob[LGT_SIZES] = { 230, 230 };
+#endif // CONFIG_LGT_FROM_PRED
+
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
static const aom_prob
default_intra_filter_probs[INTRA_FILTERS + 1][INTRA_FILTERS - 1] = {
@@ -5798,6 +5815,10 @@
#if CONFIG_FILTER_INTRA
av1_copy(fc->filter_intra_probs, default_filter_intra_probs);
#endif // CONFIG_FILTER_INTRA
+#if CONFIG_LGT_FROM_PRED
+ av1_copy(fc->intra_lgt_prob, default_intra_lgt_prob);
+ av1_copy(fc->inter_lgt_prob, default_inter_lgt_prob);
+#endif // CONFIG_LGT_FROM_PRED
#if CONFIG_LOOP_RESTORATION
av1_copy(fc->switchable_restore_prob, default_switchable_restore_prob);
#endif // CONFIG_LOOP_RESTORATION
@@ -6005,6 +6026,23 @@
fc->skip_probs[i] =
av1_mode_mv_merge_probs(pre_fc->skip_probs[i], counts->skip[i]);
+#if CONFIG_LGT_FROM_PRED
+ int j;
+ if (LGT_FROM_PRED_INTRA) {
+ for (i = TX_4X4; i < LGT_SIZES; ++i) {
+ for (j = 0; j < INTRA_MODES; ++j)
+ fc->intra_lgt_prob[i][j] = av1_mode_mv_merge_probs(
+ pre_fc->intra_lgt_prob[i][j], counts->intra_lgt[i][j]);
+ }
+ }
+ if (LGT_FROM_PRED_INTER) {
+ for (i = TX_4X4; i < LGT_SIZES; ++i) {
+ fc->inter_lgt_prob[i] = av1_mode_mv_merge_probs(pre_fc->inter_lgt_prob[i],
+ counts->inter_lgt[i]);
+ }
+ }
+#endif // CONFIG_LGT_FROM_PRED
+
if (cm->seg.temporal_update) {
for (i = 0; i < PREDICTION_PROBS; i++)
fc->seg.pred_probs[i] = av1_mode_mv_merge_probs(pre_fc->seg.pred_probs[i],
