commit d64e92c1cf3b177a5144c078b6dc77b610ca97e3
author Debargha Mukherjee <debargha@google.com> Wed Dec 20 22:23:35 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Thu Dec 21 15:19:12 2017 +0000
tree 68031ce951bc5c9eb2dc6a49dec28724eca761c4
parent 8d10183371d14e02f34a04477ca26cb67b118e39

Disable txfm domain dist for 64-pt transforms

Any 64-pt transforms only preserves half the coefficients.
Therefore transform domain distortion is no longer valid
for these transform sizes.

BUG=aomedia:1170

Change-Id: Ib99fad5497c766e2b9d2b432640c03cd49c99f50

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 7db71fc..5e62b17 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1908,6 +1908,12 @@
 #endif  // CONFIG_DIST_8X8
 
   if (cpi->sf.use_transform_domain_distortion
+#if CONFIG_TX64X64
+      // Any 64-pt transforms only preserves half the coefficients.
+      // Therefore transform domain distortion is not valid for these
+      // transform sizes.
+      && txsize_sqr_up_map[tx_size] != TX_64X64
+#endif  // CONFIG_TX64X64
 #if CONFIG_DIST_8X8
       && !x->using_dist_8x8
 #endif