commit 952eae2918bdc1ac058049f73072a0fe6a089fea
author Yushin Cho <ycho@mozilla.com> Tue Oct 03 16:21:06 2017 -0700
committer Yushin Cho <ycho@mozilla.com> Tue Oct 10 16:55:21 2017 +0000
tree 9fe88619cd3f5a4fefa241f87500704a29adaae7
parent 2b2ad0fa1303abef6b5c14e4f4bfbae2286e79c4

Use pixel domain skip error if possible in var-tx

When early skipped in var-tx, distortion is set the same as sse.
If so, use pixel domain sse (i.e. skip error) since is more accureate
than sse from transform domain.

Change-Id: Id3cbc66ea6318108c031413646f3d06250e75e7e

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 1648cf9..01fa606 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4468,7 +4468,7 @@
                    a, l, 1);
   } else {
     rd_stats->rate += rd_stats->zero_rate;
-    rd_stats->dist += RIGHT_SIGNED_SHIFT(tmp_sse, shift);
+    rd_stats->dist += tmp << 4;
     rd_stats->skip = 1;
     rd_stats->invalid_rate = 1;
     return;