commit 96c103d3b583b2c6beb1afbcc3d17c3dd66faafb
author Jingning Han <jingning@google.com> Wed Aug 05 00:55:36 2020 -0700
committer Jingning Han <jingning@google.com> Wed Aug 05 16:53:24 2020 +0000
tree a6d4f4269c6b6aa46a0feb4228b9a39f7ac6e499
parent ef79fe4dfa32e68a050b49b221d81f83075cfcc4

Properly scale diff in temporal filter

Scale the pixel difference in temporal filtering according to its
bit depth. This resolves a compression performance regression in
12 bit setting.

BUG=aomedia:2780

STATS_CHANGED

Change-Id: Id858228458a1198b558596e3c77ef4fe18d1e461

av1/encoder/temporal_filter.c

diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index d98439c..f7abc4d 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -582,7 +582,7 @@
         }
 
         // Scale down the difference for high bit depth input.
-        if (mbd->bd > 8) sum_square_diff >>= (mbd->bd - 8) * (mbd->bd - 8);
+        if (mbd->bd > 8) sum_square_diff >>= ((mbd->bd - 8) * 2);
 
         // Combine window error and block error, and normalize it.
         const double window_error = (double)sum_square_diff / num_ref_pixels;

av1/encoder/x86/highbd_temporal_filter_sse2.c

diff --git a/av1/encoder/x86/highbd_temporal_filter_sse2.c b/av1/encoder/x86/highbd_temporal_filter_sse2.c
index 8e6e43d..b7f8533 100644
--- a/av1/encoder/x86/highbd_temporal_filter_sse2.c
+++ b/av1/encoder/x86/highbd_temporal_filter_sse2.c
@@ -214,7 +214,7 @@
       }
 
       // Scale down the difference for high bit depth input.
-      diff_sse >>= (bd - 8) * (bd - 8);
+      diff_sse >>= ((bd - 8) * 2);
 
       const double window_error = (double)(diff_sse) / num_ref_pixels;
       const int subblock_idx =