commit a9ebbcc33836d8666f98de7a6d416a6dcd0cae9a
author John Koleszar <jkoleszar@google.com> Thu Apr 18 13:05:38 2013 -0700
committer John Koleszar <jkoleszar@google.com> Thu Apr 18 13:57:59 2013 -0700
tree 466865fec91567c8c72481c194ad9b1c6b8d5db8
parent ff3f93639c147fa9b8dcf6d014b3c80bf3b7c8ef

convolve: support larger blocks, fix asm saturation bug

Updates the common convoloution code to support blocks larger than
16x16, and rectangular blocks. This uncovered a bug in the SSSE3
filtering routines due to the order of application of saturation.
This commit fixes that bug, adjusts the unit test to bias its
random values towards the extremes, and adds a test to ensure that
all filters conform to the expected pairwise addition structure.

Change-Id: I81f69668b1de0de5a8ed43f0643845641525c8f0

vp9/common/vp9_convolve.c

diff --git a/vp9/common/vp9_convolve.c b/vp9/common/vp9_convolve.c
index 3ab8bec..a27ca6f 100644
--- a/vp9/common/vp9_convolve.c
+++ b/vp9/common/vp9_convolve.c
@@ -331,14 +331,14 @@
                        const int16_t *filter_y, int y_step_q4,
                        int w, int h, int taps) {
   /* Fixed size intermediate buffer places limits on parameters.
-   * Maximum intermediate_height is 39, for y_step_q4 == 32,
-   * h == 16, taps == 8.
+   * Maximum intermediate_height is 135, for y_step_q4 == 32,
+   * h == 64, taps == 8.
    */
-  uint8_t temp[16 * 39];
+  uint8_t temp[64 * 135];
   int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
 
-  assert(w <= 16);
-  assert(h <= 16);
+  assert(w <= 64);
+  assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
 
@@ -346,10 +346,10 @@
     intermediate_height = h;
 
   convolve_horiz_c(src - src_stride * (taps / 2 - 1), src_stride,
-                   temp, 16,
+                   temp, 64,
                    filter_x, x_step_q4, filter_y, y_step_q4,
                    w, intermediate_height, taps);
-  convolve_vert_c(temp + 16 * (taps / 2 - 1), 16, dst, dst_stride,
+  convolve_vert_c(temp + 64 * (taps / 2 - 1), 64, dst, dst_stride,
                   filter_x, x_step_q4, filter_y, y_step_q4,
                   w, h, taps);
 }
@@ -360,14 +360,14 @@
                            const int16_t *filter_y, int y_step_q4,
                            int w, int h, int taps) {
   /* Fixed size intermediate buffer places limits on parameters.
-   * Maximum intermediate_height is 39, for y_step_q4 == 32,
-   * h == 16, taps == 8.
+   * Maximum intermediate_height is 135, for y_step_q4 == 32,
+   * h == 64, taps == 8.
    */
-  uint8_t temp[16 * 39];
+  uint8_t temp[64 * 135];
   int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;
 
-  assert(w <= 16);
-  assert(h <= 16);
+  assert(w <= 64);
+  assert(h <= 64);
   assert(taps <= 8);
   assert(y_step_q4 <= 32);
 
@@ -375,10 +375,10 @@
     intermediate_height = h;
 
   convolve_horiz_c(src - src_stride * (taps / 2 - 1), src_stride,
-                   temp, 16,
+                   temp, 64,
                    filter_x, x_step_q4, filter_y, y_step_q4,
                    w, intermediate_height, taps);
-  convolve_avg_vert_c(temp + 16 * (taps / 2 - 1), 16, dst, dst_stride,
+  convolve_avg_vert_c(temp + 64 * (taps / 2 - 1), 64, dst, dst_stride,
                       filter_x, x_step_q4, filter_y, y_step_q4,
                       w, h, taps);
 }
@@ -563,16 +563,16 @@
                          const int16_t *filter_y, int y_step_q4,
                          int w, int h) {
   /* Fixed size intermediate buffer places limits on parameters. */
-  DECLARE_ALIGNED_ARRAY(16, uint8_t, temp, 16 * 16);
-  assert(w <= 16);
-  assert(h <= 16);
+  DECLARE_ALIGNED_ARRAY(16, uint8_t, temp, 64 * 64);
+  assert(w <= 64);
+  assert(h <= 64);
 
   vp9_convolve8(src, src_stride,
-                temp, 16,
+                temp, 64,
                 filter_x, x_step_q4,
                 filter_y, y_step_q4,
                 w, h);
-  vp9_convolve_avg(temp, 16,
+  vp9_convolve_avg(temp, 64,
                    dst, dst_stride,
                    NULL, 0, /* These unused parameter should be removed! */
                    NULL, 0, /* These unused parameter should be removed! */