Add computation of edge strength to each block

Change-Id: I9a3f29d47c3986e5931e916ec0ad03cd6aa7bad2
diff --git a/test/edge_detect_test.cc b/test/edge_detect_test.cc
index 3c8edf0..47466cb 100644
--- a/test/edge_detect_test.cc
+++ b/test/edge_detect_test.cc
@@ -9,6 +9,7 @@
  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
  */
 
+#include <stdbool.h>
 #include "aom_mem/aom_mem.h"
 #include "av1/encoder/rdopt.h"
 #include "test/util.h"
@@ -19,42 +20,29 @@
 using ::testing::get;
 using ::testing::tuple;
 
+static int get_pix(uint8_t *buf, int i, bool high_bd) {
+  if (high_bd) {
+    return *CONVERT_TO_SHORTPTR(buf + i);
+  } else {
+    return buf[i];
+  }
+}
+
 /** Get the (i, j) value from the input; if i or j is outside of the width
  * or height, the nearest pixel value is returned.
  */
-
-static int get_nearest_pix(const uint8_t *buf, int w, int h, int i, int j,
-                           int bd) {
+static int get_nearest_pix(const int *buf, int w, int h, int i, int j) {
   int offset = AOMMAX(AOMMIN(i, w - 1), 0) + w * AOMMAX(AOMMIN(j, h - 1), 0);
-  if (bd <= 8) {
-    return buf[offset];
-  } else {
-    return *CONVERT_TO_SHORTPTR(buf + offset);
-  }
-}
-
-static int get_pix(uint8_t *buf, int i, int bd) {
-  if (bd <= 8) {
-    return buf[i];
-  } else {
-    return *CONVERT_TO_SHORTPTR(buf + i);
-  }
-}
-
-static void set_pix(uint8_t *buf, int i, int v, int bd) {
-  if (bd <= 8) {
-    buf[i] = v;
-  } else {
-    *CONVERT_TO_SHORTPTR(buf + i) = v;
-  }
+  return buf[offset];
 }
 
 /** Given the image data, creates a new image with padded values, so an
  * 8-tap filter can be convolved. The padded value is the same as the closest
  * value in the image. Returns a pointer to the start of the image in the
- * padded data. Must be freed with free_pad_8tap.
+ * padded data. Must be freed with free_pad_8tap. The output will be either
+ * 8-bit or 16-bit, depending on the high bit-depth (high_bd) field.
  */
-static uint8_t *pad_8tap_convolve(const uint8_t *data, int w, int h, int bd) {
+static uint8_t *pad_8tap_convolve(const int *data, int w, int h, bool high_bd) {
   // SIMD optimizations require the width to be a multiple of 8 and the height
   // to be multiples of 4.
   assert(w % 8 == 0);
@@ -65,20 +53,19 @@
   const int pad_h = h + 7;
 
   uint8_t *dst;
-  if (bd <= 8) {
-    dst = (uint8_t *)aom_memalign(32, sizeof(uint8_t) * pad_w * pad_h);
-  } else {
+  if (high_bd) {
     dst =
         CONVERT_TO_BYTEPTR(aom_memalign(32, sizeof(uint16_t) * pad_w * pad_h));
+  } else {
+    dst = (uint8_t *)aom_memalign(32, sizeof(uint8_t) * pad_w * pad_h);
   }
-
   for (int j = 0; j < pad_h; ++j) {
     for (int i = 0; i < pad_w; ++i) {
-      const int v = get_nearest_pix(data, w, h, i - 3, j - 3, bd);
-      if (bd <= 8) {
-        dst[i + j * pad_w] = v;
-      } else {
+      const int v = get_nearest_pix(data, w, h, i - 3, j - 3);
+      if (high_bd) {
         *CONVERT_TO_SHORTPTR(dst + i + j * pad_w) = v;
+      } else {
+        dst[i + j * pad_w] = v;
       }
     }
   }
@@ -87,116 +74,119 @@
 
 static int stride_8tap(int width) { return width + 7; }
 
-static void free_pad_8tap(uint8_t *padded, int width, int bd) {
-  if (bd <= 8) {
-    aom_free(padded - (width + 7) * 3 - 3);
-  } else {
+static void free_pad_8tap(uint8_t *padded, int width, bool high_bd) {
+  if (high_bd) {
     aom_free(CONVERT_TO_SHORTPTR(padded - (width + 7) * 3 - 3));
+  } else {
+    aom_free(padded - (width + 7) * 3 - 3);
   }
 }
 
-static uint8_t *malloc_bd(int num_entries, int bd) {
-  const int bytes_per_entry = bd <= 8 ? sizeof(uint8_t) : sizeof(uint16_t);
+static uint8_t *malloc_bd(int num_entries, bool high_bd) {
+  const int bytes_per_entry = high_bd ? sizeof(uint16_t) : sizeof(uint8_t);
 
   uint8_t *buf = (uint8_t *)aom_memalign(32, bytes_per_entry * num_entries);
-  if (bd <= 8) {
-    return buf;
-  } else {
+  if (high_bd) {
     return CONVERT_TO_BYTEPTR(buf);
+  } else {
+    return buf;
   }
 }
 
-static void free_bd(uint8_t *p, int bd) {
-  if (bd <= 8) {
-    aom_free(p);
-  } else {
+static void free_bd(uint8_t *p, bool high_bd) {
+  if (high_bd) {
     aom_free(CONVERT_TO_SHORTPTR(p));
+  } else {
+    aom_free(p);
   }
 }
 
 class EdgeDetectBrightnessTest :
-    // Parameters are (brightness, width, height, bit depth).
-    public ::testing::TestWithParam<tuple<int, int, int, int> > {
+    // Parameters are (brightness, width, height, high bit depth representation,
+    // bit depth).
+    public ::testing::TestWithParam<tuple<int, int, int, bool, int> > {
  protected:
   void SetUp() override {
     // Allocate a (width by height) array of luma values in orig_.
     // padded_ will be filled by the pad() call, which adds a border around
     // the orig_. The output_ array has enough space for the computation.
+    const int brightness = GET_PARAM(0);
     const int width = GET_PARAM(1);
     const int height = GET_PARAM(2);
-    const int bd = GET_PARAM(3);
-    orig_ = malloc_bd(width * height, bd);
-    padded_ = nullptr;
-    output_ = malloc_bd(width * height, bd);
+    const bool high_bd = GET_PARAM(3);
+
+    // Create the padded image of uniform brightness.
+    int *orig = (int *)malloc(width * height * sizeof(int));
+    for (int i = 0; i < width * height; ++i) {
+      orig[i] = brightness;
+    }
+    input_ = pad_8tap_convolve(orig, width, height, high_bd);
+    free(orig);
+    output_ = malloc_bd(width * height, high_bd);
   }
 
   void TearDown() override {
-    const int bd = GET_PARAM(3);
-    if (orig_ != nullptr) {
-      free_bd(orig_, bd);
-    }
-    if (padded_ != nullptr) {
-      const int width = GET_PARAM(1);
-      free_pad_8tap(padded_, width, bd);
-    }
-    free_bd(output_, bd);
-  }
-
-  void pad() {
     const int width = GET_PARAM(1);
-    const int height = GET_PARAM(2);
-    const int bd = GET_PARAM(3);
-    padded_ = pad_8tap_convolve(orig_, width, height, bd);
-    // Get rid of the original buffer, it should not be used further.
-    free_bd(orig_, bd);
-    orig_ = nullptr;
+    const bool high_bd = GET_PARAM(3);
+    free_pad_8tap(input_, width, high_bd);
+    free_bd(output_, high_bd);
   }
 
-  uint8_t *orig_;
-  uint8_t *padded_;
+  // Skip the tests where brightness exceeds the bit-depth; we run into this
+  // issue because of gtest's limitation on valid combinations of test
+  // parameters. Also skip the tests where bit depth is greater than 8, but
+  // high bit depth representation is not set.
+  bool should_skip() const {
+    const int brightness = GET_PARAM(0);
+    const int bd = GET_PARAM(4);
+    if (brightness >= (1 << bd)) {
+      return true;
+    }
+    const bool high_bd = GET_PARAM(3);
+    if (bd > 8 && !high_bd) {
+      return true;
+    }
+    return false;
+  }
+
+  uint8_t *input_;
   uint8_t *output_;
 };
 
 TEST_P(EdgeDetectBrightnessTest, BlurUniformBrightness) {
+  // Some combination of parameters are non-sensical, due to limitations
+  // of the testing framework. Ignore these.
+  if (should_skip()) {
+    return;
+  }
+
   // For varying levels of brightness, the algorithm should
   // produce the same output.
   const int brightness = GET_PARAM(0);
   const int width = GET_PARAM(1);
   const int height = GET_PARAM(2);
-  const int bd = GET_PARAM(3);
-  // Skip the tests where brightness exceeds the bit-depth; we run into this
-  // issue because of gtest's limitation on valid combinations of test
-  // parameters.
-  if (brightness >= (1 << bd)) {
-    return;
-  }
+  const bool high_bd = GET_PARAM(3);
+  const int bd = GET_PARAM(4);
+
+  gaussian_blur(input_, stride_8tap(width), width, height, output_, high_bd,
+                bd);
   for (int i = 0; i < width * height; ++i) {
-    set_pix(orig_, i, brightness, bd);
-  }
-  pad();
-  gaussian_blur(padded_, stride_8tap(width), width, height, output_, bd);
-  for (int i = 0; i < width * height; ++i) {
-    ASSERT_EQ(brightness, get_pix(output_, i, bd));
+    ASSERT_EQ(brightness, get_pix(output_, i, high_bd));
   }
 }
 
 // No edges on a uniformly bright image.
 TEST_P(EdgeDetectBrightnessTest, DetectUniformBrightness) {
-  const int brightness = GET_PARAM(0);
-  const int width = GET_PARAM(1);
-  const int height = GET_PARAM(2);
-  const int bd = GET_PARAM(3);
-  // Skip the tests where brightness exceeds the bit-depth; we run into this
-  // issue because of gtest's limitation on valid combinations of test
-  // parameters.
-  if (brightness >= (1 << bd)) {
+  if (should_skip()) {
     return;
   }
-  for (int i = 0; i < width * height; ++i) {
-    set_pix(orig_, i, brightness, bd);
-  }
-  pad();
-  ASSERT_EQ(0, av1_edge_exists(padded_, stride_8tap(width), width, height, bd));
+  const int width = GET_PARAM(1);
+  const int height = GET_PARAM(2);
+  const bool high_bd = GET_PARAM(3);
+  const int bd = GET_PARAM(4);
+
+  ASSERT_EQ(0, av1_edge_exists(input_, stride_8tap(width), width, height,
+                               high_bd, bd));
 }
 
 INSTANTIATE_TEST_CASE_P(ImageBrightnessTests, EdgeDetectBrightnessTest,
@@ -209,138 +199,132 @@
                             ::testing::Values(8, 16, 32),
                             // Height
                             ::testing::Values(4, 8, 12, 32),
+                            // High bit depth representation
+                            ::testing::Bool(),
                             // Bit depth
                             ::testing::Values(8, 10, 12)));
 
 class EdgeDetectImageTest :
-    // Parameters are (width, height, bit depth).
-    public ::testing::TestWithParam<tuple<int, int, int> > {};
+    // Parameters are (width, height, high bit depth representation, bit depth).
+    public ::testing::TestWithParam<tuple<int, int, bool, int> > {
+ protected:
+  // Skip the tests where bit depth is greater than 8, but high bit depth
+  // representation is not set (limitation of testing framework).
+  bool should_skip() const {
+    const bool high_bd = GET_PARAM(2);
+    const int bd = GET_PARAM(3);
+    return bd > 8 && !high_bd;
+  }
+};
 
 // Generate images with black on one side and white on the other.
 TEST_P(EdgeDetectImageTest, BlackWhite) {
+  // Some combination of parameters are non-sensical, due to limitations
+  // of the testing framework. Ignore these.
+  if (should_skip()) {
+    return;
+  }
+
   const int width = GET_PARAM(0);
   const int height = GET_PARAM(1);
-  const int bd = GET_PARAM(2);
+  const bool high_bd = GET_PARAM(2);
+  const int bd = GET_PARAM(3);
+
   const int white = (1 << bd) - 1;
-  uint8_t *orig = malloc_bd(width * height, bd);
+  int *orig = (int *)malloc(width * height * sizeof(int));
   for (int j = 0; j < height; ++j) {
     for (int i = 0; i < width; ++i) {
       if (i < width / 2) {
-        set_pix(orig, i + j * width, 0, bd);
+        orig[i + j * width] = 0;
       } else {
-        set_pix(orig, i + j * width, white, bd);
+        orig[i + j * width] = white;
       }
     }
   }
-  uint8_t *padded = pad_8tap_convolve(orig, width, height, bd);
-  free_bd(orig, bd);
+  uint8_t *padded = pad_8tap_convolve(orig, width, height, high_bd);
+  free(orig);
   // Value should be between 556 and 560.
-  ASSERT_LE(556,
-            av1_edge_exists(padded, stride_8tap(width), width, height, bd));
-  ASSERT_GE(560,
-            av1_edge_exists(padded, stride_8tap(width), width, height, bd));
+  ASSERT_LE(556, av1_edge_exists(padded, stride_8tap(width), width, height,
+                                 high_bd, bd));
+  ASSERT_GE(560, av1_edge_exists(padded, stride_8tap(width), width, height,
+                                 high_bd, bd));
 
-  free_pad_8tap(padded, width, bd);
+  free_pad_8tap(padded, width, high_bd);
 }
 
 // Hardcoded blur tests.
-static const uint8_t luma[32] = { 241, 147, 7,   90,  184, 103, 28,  186,
-                                  2,   248, 49,  242, 114, 146, 127, 22,
-                                  121, 228, 167, 108, 158, 174, 41,  168,
-                                  214, 99,  184, 109, 114, 247, 117, 119 };
+static const int luma[32] = { 241, 147, 7,   90,  184, 103, 28,  186,
+                              2,   248, 49,  242, 114, 146, 127, 22,
+                              121, 228, 167, 108, 158, 174, 41,  168,
+                              214, 99,  184, 109, 114, 247, 117, 119 };
 static const uint8_t expected[] = { 161, 138, 119, 118, 123, 118, 113, 122,
                                     143, 140, 134, 133, 134, 126, 116, 114,
                                     147, 149, 145, 142, 143, 138, 126, 118,
                                     164, 156, 148, 144, 148, 148, 138, 126 };
 
-TEST(EdgeDetectImageTest, HardcodedBlurTest) {
+static void hardcoded_blur_test_aux(const bool high_bd) {
   const int w = 8;
   const int h = 4;
-  int bd = 8;
-  uint8_t *output = malloc_bd(w * h, bd);
-  uint8_t *padded = pad_8tap_convolve(luma, w, h, bd);
-  gaussian_blur(padded, stride_8tap(w), w, h, output, bd);
-  for (int i = 0; i < w * h; ++i) {
-    ASSERT_EQ(expected[i], get_pix(output, i, bd));
-  }
-  free_pad_8tap(padded, w, bd);
-  free_bd(output, bd);
-
-  // High bit-depth tests.
-  for (bd = 10; bd <= 12; bd += 2) {
-    uint16_t luma16[32];
-    for (int i = 0; i < 32; ++i) {
-      luma16[i] = luma[i];
+  for (int bd = 8; bd <= 12; bd += 2) {
+    // Skip the tests where bit depth is greater than 8, but high bit depth
+    // representation is not set.
+    if (bd > 8 && !high_bd) {
+      break;
     }
-    uint8_t *output = malloc_bd(w * h, bd);
-    uint8_t *padded = pad_8tap_convolve(CONVERT_TO_BYTEPTR(luma16), w, h, bd);
-    gaussian_blur(padded, stride_8tap(w), w, h, output, bd);
+    uint8_t *output = malloc_bd(w * h, high_bd);
+    uint8_t *padded = pad_8tap_convolve(luma, w, h, high_bd);
+    gaussian_blur(padded, stride_8tap(w), w, h, output, high_bd, bd);
     for (int i = 0; i < w * h; ++i) {
-      ASSERT_EQ(expected[i], get_pix(output, i, bd));
+      ASSERT_EQ(expected[i], get_pix(output, i, high_bd));
     }
-    free_pad_8tap(padded, w, bd);
-    free_bd(output, bd);
-  }
-  // If we multiply the inputs by a constant factor, the output should not vary
-  // more than 0.5 * factor.
-  for (bd = 10; bd <= 12; bd += 2) {
+    free_pad_8tap(padded, w, high_bd);
+    free_bd(output, high_bd);
+
+    // If we multiply the inputs by a constant factor, the output should not
+    // vary more than 0.5 * factor.
     for (int c = 2; c < (1 << (bd - 8)); ++c) {
-      uint16_t luma16[32];
+      int scaled_luma[32];
       for (int i = 0; i < 32; ++i) {
-        luma16[i] = luma[i] * c;
+        scaled_luma[i] = luma[i] * c;
       }
-      uint8_t *output = malloc_bd(w * h, bd);
-      uint8_t *padded = pad_8tap_convolve(CONVERT_TO_BYTEPTR(luma16), w, h, bd);
-      gaussian_blur(padded, stride_8tap(w), w, h, output, bd);
+      uint8_t *output = malloc_bd(w * h, high_bd);
+      uint8_t *padded = pad_8tap_convolve(scaled_luma, w, h, high_bd);
+      gaussian_blur(padded, stride_8tap(w), w, h, output, high_bd, bd);
       for (int i = 0; i < w * h; ++i) {
-        ASSERT_GE(c / 2, abs(expected[i] * c - get_pix(output, i, bd)));
+        ASSERT_GE(c / 2, abs(expected[i] * c - get_pix(output, i, high_bd)));
       }
-      free_pad_8tap(padded, w, bd);
-      free_bd(output, bd);
+      free_pad_8tap(padded, w, high_bd);
+      free_bd(output, high_bd);
     }
   }
 }
 
-TEST(EdgeDetectImageTest, HardcodedHighBdBlurTest) {
-  // Randomly generated 8x4.
-  const uint16_t luma[32] = { 241, 147, 7,   90,  184, 103, 28,  186,
-                              2,   248, 49,  242, 114, 146, 127, 22,
-                              121, 228, 167, 108, 158, 174, 41,  168,
-                              214, 99,  184, 109, 114, 247, 117, 119 };
-  uint16_t expected[] = { 161, 138, 119, 118, 123, 118, 113, 122, 143, 140, 134,
-                          133, 134, 126, 116, 114, 147, 149, 145, 142, 143, 138,
-                          126, 118, 164, 156, 148, 144, 148, 148, 138, 126 };
-  const int w = 8;
-  const int h = 4;
-  for (int bd = 10; bd <= 12; bd += 2) {
-    uint8_t *padded = pad_8tap_convolve(CONVERT_TO_BYTEPTR(luma), w, h, bd);
-    uint8_t *output = malloc_bd(w * h, bd);
-    gaussian_blur(padded, stride_8tap(w), w, h, output, bd);
-
-    for (int i = 0; i < w * h; ++i) {
-      ASSERT_EQ(expected[i], get_pix(output, i, bd));
-    }
-    free_pad_8tap(padded, w, bd);
-    free_bd(output, bd);
-  }
+TEST(EdgeDetectImageTest, HardcodedBlurTest) {
+  hardcoded_blur_test_aux(false);
+  hardcoded_blur_test_aux(true);
 }
 
 TEST(EdgeDetectImageTest, SobelTest) {
   // Randomly generated 3x3. Compute Sobel for middle value.
   const uint8_t buf[9] = { 241, 147, 7, 90, 184, 103, 28, 186, 2 };
   const int stride = 3;
-  int bd = 8;
-  sobel_xy result = sobel(buf, stride, 1, 1, bd);
+  bool high_bd = false;
+  sobel_xy result = sobel(buf, stride, 1, 1, high_bd);
+  ASSERT_EQ(234, result.x);
+  ASSERT_EQ(140, result.y);
+
+  // Verify it works for 8-bit values in a high bit-depth buffer.
+  const uint16_t buf8_16[9] = { 241, 147, 7, 90, 184, 103, 28, 186, 2 };
+  high_bd = true;
+  result = sobel(CONVERT_TO_BYTEPTR(buf8_16), stride, 1, 1, high_bd);
   ASSERT_EQ(234, result.x);
   ASSERT_EQ(140, result.y);
 
   // Verify it works for high bit-depth values as well.
   const uint16_t buf16[9] = { 241, 147, 7, 90, 184, 2003, 1028, 186, 2 };
-  for (bd = 10; bd <= 12; bd += 2) {
-    result = sobel(CONVERT_TO_BYTEPTR(buf16), stride, 1, 1, bd);
-    ASSERT_EQ(-2566, result.x);
-    ASSERT_EQ(-860, result.y);
-  }
+  result = sobel(CONVERT_TO_BYTEPTR(buf16), stride, 1, 1, high_bd);
+  ASSERT_EQ(-2566, result.x);
+  ASSERT_EQ(-860, result.y);
 }
 
 INSTANTIATE_TEST_CASE_P(EdgeDetectImages, EdgeDetectImageTest,
@@ -349,6 +333,8 @@
                             ::testing::Values(8, 16, 32),
                             // Height
                             ::testing::Values(4, 8, 12, 32),
+                            // High bit depth representation
+                            ::testing::Bool(),
                             // Bit depth
                             ::testing::Values(8, 10, 12)));
 }  // namespace