lgt-from-pred: transforms based on prediction
In this experiment, sharp image discontinuity in the predicted
block is detected. Based on this discontinuity, we choose
particular LGTs as row and column transforms.
Bitstream syntax, entropy coding, and RD search for LGT are added.
One binary symbol is used to signal whether LGT is used. This
experiment can work independently with the lgt experiment.
lowres: -0.414% for key frames, -0.151% overall
midres: -0.413% for key frames, -0.161% overall
Change-Id: Iaa2f2c2839c34ca4134fa55e77870dc3f1fa879f
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index ffd3680..30ad337 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -385,6 +385,9 @@
#if CONFIG_TXK_SEL
TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif
+#if CONFIG_LGT_FROM_PRED
+ int use_lgt;
+#endif
#if CONFIG_FILTER_INTRA
FILTER_INTRA_MODE_INFO filter_intra_mode_info;
@@ -1053,6 +1056,36 @@
return av1_num_ext_tx_set[set_type];
}
+#if CONFIG_LGT_FROM_PRED
+static INLINE int is_lgt_allowed(PREDICTION_MODE mode, TX_SIZE tx_size) {
+ if (!LGT_FROM_PRED_INTRA && !is_inter_mode(mode)) return 0;
+ if (!LGT_FROM_PRED_INTER && is_inter_mode(mode)) return 0;
+
+ switch (mode) {
+ case D45_PRED:
+ case D63_PRED:
+ case D117_PRED:
+ case V_PRED:
+#if CONFIG_SMOOTH_HV
+ case SMOOTH_V_PRED:
+#endif
+ return tx_size_wide[tx_size] <= 8;
+ case D135_PRED:
+ case D153_PRED:
+ case D207_PRED:
+ case H_PRED:
+#if CONFIG_SMOOTH_HV
+ case SMOOTH_H_PRED:
+#endif
+ return tx_size_high[tx_size] <= 8;
+ case DC_PRED:
+ case SMOOTH_PRED: return 0;
+ case TM_PRED:
+ default: return tx_size_wide[tx_size] <= 8 || tx_size_high[tx_size] <= 8;
+ }
+}
+#endif // CONFIG_LGT_FROM_PRED
+
#if CONFIG_RECT_TX
static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
static const char LUT[BLOCK_SIZES_ALL] = {
