commit ad8290b8e68af7c162f2a561d20027e9d7922a4f
author Lester Lu <kslu@google.com> Mon Jun 12 18:26:18 2017 -0700
committer Lester Lu <kslu@google.com> Mon Jun 26 19:11:25 2017 +0000
tree ebb22153e8e62dbc0f859278891aefe3ae1c1387
parent 75b0100431e5914e4be3f92635afa5a73615e71a

New experiment: LGT

In previous ADSTs, DST-7 and DST-4 are used for length 4 and length
8/16/32, respectively. In this LGT experiment we explore transforms
between DST-4 and DST-7. When CONFIG_LGT flag is on, adst4 and adst8
are replaced by lgt4 and lgt8, the intermediate transforms with
pre-chosen parameters.

The LGTs applied here are lgt4_160 and lgt8_170, where the numbers
mean the self-loop weights times 100. The associated values for DST-7
and DST-4 are 100 and 200.

ovr_psnr:
lowres: -0.140
midres: -0.131
hdres: -0.078

These changes are not applied to the highbd scenario in the
current version.

Change-Id: I20600456da8766528b2b6b11aa28801e70af498e

av1/encoder/hybrid_fwd_txfm.c

diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c
index c71cae9..a5409c2 100644
--- a/av1/encoder/hybrid_fwd_txfm.c
+++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -52,35 +52,56 @@
     return;
   }
 
+#if CONFIG_LGT
+  // only C version has LGTs
+  av1_fht4x4_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht4x4(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_4x8(const int16_t *src_diff, tran_low_t *coeff,
                          int diff_stride, TX_TYPE tx_type,
                          FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht4x8_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht4x8(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_8x4(const int16_t *src_diff, tran_low_t *coeff,
                          int diff_stride, TX_TYPE tx_type,
                          FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht8x4_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht8x4(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_8x16(const int16_t *src_diff, tran_low_t *coeff,
                           int diff_stride, TX_TYPE tx_type,
                           FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht8x16_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht8x16(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_16x8(const int16_t *src_diff, tran_low_t *coeff,
                           int diff_stride, TX_TYPE tx_type,
                           FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht16x8_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht16x8(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_16x32(const int16_t *src_diff, tran_low_t *coeff,
@@ -101,7 +122,11 @@
                          int diff_stride, TX_TYPE tx_type,
                          FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht8x8_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht8x8(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff,
@@ -137,28 +162,44 @@
                           int diff_stride, TX_TYPE tx_type,
                           FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht16x4_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht16x4(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_4x16(const int16_t *src_diff, tran_low_t *coeff,
                           int diff_stride, TX_TYPE tx_type,
                           FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht4x16_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht4x16(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_32x8(const int16_t *src_diff, tran_low_t *coeff,
                           int diff_stride, TX_TYPE tx_type,
                           FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht32x8_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht32x8(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 
 static void fwd_txfm_8x32(const int16_t *src_diff, tran_low_t *coeff,
                           int diff_stride, TX_TYPE tx_type,
                           FWD_TXFM_OPT fwd_txfm_opt) {
   (void)fwd_txfm_opt;
+#if CONFIG_LGT
+  av1_fht8x32_c(src_diff, coeff, diff_stride, tx_type);
+#else
   av1_fht8x32(src_diff, coeff, diff_stride, tx_type);
+#endif
 }
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT