commit 0e840e7580051e069dc19225323fed3a04e7a0d8
author Frank Bossen <fbossen@gmail.com> Tue Feb 27 20:29:00 2018 +0100
committer Yaowu Xu <yaowu@google.com> Wed Feb 28 21:54:06 2018 +0000
tree 874ad2bb0829b400f29d91c2d9d51bb7709c553b
parent 993f7be04d60896825f67bbab87f4640eca7693d

[normative][txfm] Remove clamping in idst4

bug=aomedia:1447

Change-Id: I4b963e0f21766e1ffa269fe50d7c4ec6c9b2f21f

av1/common/av1_inv_txfm1d.c

diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c
index 57b1578..28b3f5b 100644
--- a/av1/common/av1_inv_txfm1d.c
+++ b/av1/common/av1_inv_txfm1d.c
@@ -760,12 +760,10 @@
   s4 = range_check_value(sinpi[1] * x2, stage_range[1] + bit);
   s5 = range_check_value(sinpi[2] * x3, stage_range[1] + bit);
   s6 = range_check_value(sinpi[4] * x3, stage_range[1] + bit);
-  s7 = x0 - x2;
-  if (stage_range[1] <= 16) s7 = clamp_value(s7, 16);
+  s7 = x0 - x2;  // no range check here. Range is checked in stage 3
 
   // stage 2
-  s7 = s7 + x3;
-  if (stage_range[2] <= 16) s7 = clamp_value(s7, 16);
+  s7 = s7 + x3;  // no range check here. Range is checked in stage 3
 
   // stage 3
   s0 = range_check_value(s0 + s3, stage_range[3] + bit);
@@ -786,10 +784,6 @@
   // stage 6
   x3 = range_check_value(x3 - s3, stage_range[6] + bit);
 
-  // 1-D transform scaling factor is sqrt(2).
-  // The overall dynamic range is 14b (input) + 14b (multiplication scaling)
-  // + 1b (addition) = 29b.
-  // Hence the output bit depth is 15b.
   output[0] = round_shift(x0, bit);
   output[1] = round_shift(x1, bit);
   output[2] = round_shift(x2, bit);