commit 708c1ec5abf92c3762be1359f7340b39afcc5412
author Lester Lu <kslu@google.com> Wed Jun 14 14:54:49 2017 -0700
committer Lester Lu <kslu@google.com> Mon Jul 10 19:36:24 2017 +0000
tree 5b9d9b10bfc006e244f61c95d564e0eb71ea6142
parent 7651b7394ce800ab8c2feeae0454f86de9904cf6

Inter and intra LGTs

Here we have an LGT to replace ADST for intra residual blocks, and
another LGT to replace ADST for inter residual blocks. The changes
are only applied to transform length 4 and 8, and only for the
lowbitdepth path.

lowres: -0.18%

Change-Id: Iadc1e02b53e3756b44f74ca648cfa8b0e8ca7af4

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index f644021..0c74676 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1694,8 +1694,11 @@
         const PLANE_TYPE plane_type = get_plane_type(plane);
         TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
 
-        av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, recon,
-                                    MAX_TX_SIZE, eob);
+        av1_inverse_transform_block(xd, dqcoeff,
+#if CONFIG_LGT
+                                    xd->mi[0]->mbmi.mode,
+#endif
+                                    tx_type, tx_size, recon, MAX_TX_SIZE, eob);
 
 #if CONFIG_DAALA_DIST
         if (plane == 0 && (bsw < 8 || bsh < 8)) {
@@ -3102,6 +3105,9 @@
             if (!skip)
 #endif
               av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block),
+#if CONFIG_LGT
+                                          mode,
+#endif
                                           DCT_DCT, tx_size, dst, dst_stride,
                                           p->eobs[block]);
           } else {
@@ -3151,6 +3157,9 @@
             if (!skip)
 #endif
               av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block),
+#if CONFIG_LGT
+                                          mode,
+#endif
                                           tx_type, tx_size, dst, dst_stride,
                                           p->eobs[block]);
             cpi->fn_ptr[sub_bsize].vf(src, src_stride, dst, dst_stride, &tmp);
@@ -3334,6 +3343,9 @@
           if (!skip)
 #endif  // CONFIG_PVQ
             av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block),
+#if CONFIG_LGT
+                                        mode,
+#endif
                                         tx_type, tx_size, dst, dst_stride,
                                         p->eobs[block]);
           unsigned int tmp;
@@ -3349,6 +3361,9 @@
           if (!skip)
 #endif  // CONFIG_PVQ
             av1_inverse_transform_block(xd, BLOCK_OFFSET(pd->dqcoeff, block),
+#if CONFIG_LGT
+                                        mode,
+#endif
                                         DCT_DCT, tx_size, dst, dst_stride,
                                         p->eobs[block]);
         }
@@ -4254,8 +4269,14 @@
   rd_stats->sse += tmp * 16;
   const int eob = p->eobs[block];
 
+#if CONFIG_LGT
+  PREDICTION_MODE mode = get_prediction_mode(xd->mi[0], plane, tx_size, block);
+  av1_inverse_transform_block(xd, dqcoeff, mode, tx_type, tx_size, rec_buffer,
+                              MAX_TX_SIZE, eob);
+#else
   av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, rec_buffer,
                               MAX_TX_SIZE, eob);
+#endif
   if (eob > 0) {
 #if CONFIG_DAALA_DIST
     if (plane == 0 && (bw < 8 && bh < 8)) {