commit c7a5e883080e245b14f618c6b2f6c004d6ddd914
author Debargha Mukherjee <debargha@google.com> Mon Feb 19 04:05:21 2018 -0800
committer Debargha Mukherjee <debargha@google.com> Mon Feb 19 04:12:15 2018 -0800
tree a2f9dc4244c01a38f89a37b9140a4d99e1494ac2
parent a840cde6176d1c5dff3c43aa46c58255b03b8e44

Fix a 12-bit convolve bug

Fixes several failing 12-bit tests.

Change-Id: If6da50bbd595adb270cb11820df97b34a22d7b82

av1/common/convolve.c

diff --git a/av1/common/convolve.c b/av1/common/convolve.c
index b10e99e..9c44041 100644
--- a/av1/common/convolve.c
+++ b/av1/common/convolve.c
@@ -1290,6 +1290,7 @@
     // 0-bit rounding just to convert from int32 to uint16
     const int rbits =
         2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
+    assert(rbits >= 0);
     av1_highbd_convolve_rounding(tmp_dst, tmp_dst_stride, dst, dst_stride, w, h,
                                  rbits, bd);
   }

av1/common/convolve.h

diff --git a/av1/common/convolve.h b/av1/common/convolve.h
index f3e0dcd..ae66367 100644
--- a/av1/common/convolve.h
+++ b/av1/common/convolve.h
@@ -105,11 +105,13 @@
   conv_params.is_compound = 0;
   conv_params.dst = NULL;
   conv_params.dst_stride = 0;
+#if CONFIG_LOWPRECISION_BLEND
   const int intbufrange = bd + FILTER_BITS - conv_params.round_0 + 2;
   if (bd < 12) assert(intbufrange <= 16);
   if (intbufrange > 16) {
     conv_params.round_0 += intbufrange - 16;
   }
+#endif  // CONFIG_LOWPRECISION_BLEND
   return conv_params;
 }
 
@@ -127,16 +129,15 @@
 #if CONFIG_LOWPRECISION_BLEND
   conv_params.round_1 = is_compound ? COMPOUND_ROUND1_BITS
                                     : 2 * FILTER_BITS - conv_params.round_0;
-#else
-  conv_params.round_1 = 0;
-#endif
   const int intbufrange = bd + FILTER_BITS - conv_params.round_0 + 2;
   if (bd < 12) assert(intbufrange <= 16);
   if (intbufrange > 16) {
     conv_params.round_0 += intbufrange - 16;
-    if (is_compound && conv_params.round_1 > 0)
-      conv_params.round_1 -= intbufrange - 16;
+    conv_params.round_1 -= intbufrange - 16;
   }
+#else
+  conv_params.round_1 = 0;
+#endif  // CONFIG_LOWPRECISION_BLEND
   // TODO(yunqing): The following dst should only be valid while
   // is_compound = 1;
   conv_params.dst = dst;