[NEON] highbd implementation of av1_highbd_convolve_y_sr_neon 3/3
Approximately 6.1x faster. This is the 3rd part of the 6-tap filter
implementation. Helper functions have been added.
Change-Id: I824060ce01c96a3062c5be8d1a490f70870280e6
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h
index e07ea80..8f76f02 100644
--- a/aom_dsp/arm/mem_neon.h
+++ b/aom_dsp/arm/mem_neon.h
@@ -324,6 +324,22 @@
*s4 = vld1_s16(s);
}
+static INLINE void load_u16_4x5(const uint16_t *s, const ptrdiff_t p,
+ uint16x4_t *const s0, uint16x4_t *const s1,
+ uint16x4_t *const s2, uint16x4_t *const s3,
+ uint16x4_t *const s4) {
+ *s0 = vld1_u16(s);
+ s += p;
+ *s1 = vld1_u16(s);
+ s += p;
+ *s2 = vld1_u16(s);
+ s += p;
+ *s3 = vld1_u16(s);
+ s += p;
+ *s4 = vld1_u16(s);
+ s += p;
+}
+
static INLINE void load_u8_8x5(const uint8_t *s, ptrdiff_t p,
uint8x8_t *const s0, uint8x8_t *const s1,
uint8x8_t *const s2, uint8x8_t *const s3,
@@ -339,6 +355,22 @@
*s4 = vld1_u8(s);
}
+static INLINE void load_u16_8x5(const uint16_t *s, const ptrdiff_t p,
+ uint16x8_t *const s0, uint16x8_t *const s1,
+ uint16x8_t *const s2, uint16x8_t *const s3,
+ uint16x8_t *const s4) {
+ *s0 = vld1q_u16(s);
+ s += p;
+ *s1 = vld1q_u16(s);
+ s += p;
+ *s2 = vld1q_u16(s);
+ s += p;
+ *s3 = vld1q_u16(s);
+ s += p;
+ *s4 = vld1q_u16(s);
+ s += p;
+}
+
static INLINE void load_s16_4x4(const int16_t *s, ptrdiff_t p,
int16x4_t *const s0, int16x4_t *const s1,
int16x4_t *const s2, int16x4_t *const s3) {
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
index 12bd22f..7118f5c 100644
--- a/av1/common/arm/highbd_convolve_neon.c
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -23,6 +23,131 @@
#include "av1/common/filter.h"
#include "av1/common/arm/highbd_convolve_neon.h"
+static INLINE void highbd_convolve_y_sr_6tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr,
+ const int dst_stride, int w, int h, const int16_t *y_filter_ptr, int bd) {
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
+
+ if (w <= 4) {
+ uint16x4_t t0, t1, t2, t3, t4, t5, t6, t7, t8;
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
+ uint16x4_t d0, d1, d2, d3;
+ uint16x8_t d01, d23;
+
+ const uint16_t *s = src_ptr + src_stride;
+ uint16_t *d = dst_ptr;
+
+ load_u16_4x5(s, src_stride, &t0, &t1, &t2, &t3, &t4);
+ s0 = vreinterpret_s16_u16(t0);
+ s1 = vreinterpret_s16_u16(t1);
+ s2 = vreinterpret_s16_u16(t2);
+ s3 = vreinterpret_s16_u16(t3);
+ s4 = vreinterpret_s16_u16(t4);
+ s += 5 * src_stride;
+
+ do {
+ load_u16_4x4(s, src_stride, &t5, &t6, &t7, &t8);
+ s5 = vreinterpret_s16_u16(t5);
+ s6 = vreinterpret_s16_u16(t6);
+ s7 = vreinterpret_s16_u16(t7);
+ s8 = vreinterpret_s16_u16(t8);
+
+ d0 = highbd_convolve6_4_s32_s16(s0, s1, s2, s3, s4, s5, y_filter_0_7);
+ d1 = highbd_convolve6_4_s32_s16(s1, s2, s3, s4, s5, s6, y_filter_0_7);
+ d2 = highbd_convolve6_4_s32_s16(s2, s3, s4, s5, s6, s7, y_filter_0_7);
+ d3 = highbd_convolve6_4_s32_s16(s3, s4, s5, s6, s7, s8, y_filter_0_7);
+
+ d01 = vcombine_u16(d0, d1);
+ d23 = vcombine_u16(d2, d3);
+
+ d01 = vminq_u16(d01, max);
+ d23 = vminq_u16(d23, max);
+
+ if (w == 2) {
+ store_u16q_2x1(d + 0 * dst_stride, d01, 0);
+ store_u16q_2x1(d + 1 * dst_stride, d01, 2);
+ if (h != 2) {
+ store_u16q_2x1(d + 2 * dst_stride, d23, 0);
+ store_u16q_2x1(d + 3 * dst_stride, d23, 2);
+ }
+ } else {
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ if (h != 2) {
+ vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
+ vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
+ }
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ // if width is a multiple of 8 & height is a multiple of 4
+ uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
+ uint16x8_t d0, d1, d2, d3;
+
+ do {
+ int height = h;
+ const uint16_t *s = src_ptr + src_stride;
+ uint16_t *d = dst_ptr;
+
+ load_u16_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4);
+ s0 = vreinterpretq_s16_u16(t0);
+ s1 = vreinterpretq_s16_u16(t1);
+ s2 = vreinterpretq_s16_u16(t2);
+ s3 = vreinterpretq_s16_u16(t3);
+ s4 = vreinterpretq_s16_u16(t4);
+ s += 5 * src_stride;
+
+ do {
+ load_u16_8x4(s, src_stride, &t5, &t6, &t7, &t8);
+ s5 = vreinterpretq_s16_u16(t5);
+ s6 = vreinterpretq_s16_u16(t6);
+ s7 = vreinterpretq_s16_u16(t7);
+ s8 = vreinterpretq_s16_u16(t8);
+
+ d0 = highbd_convolve6_8_s32_s16(s0, s1, s2, s3, s4, s5, y_filter_0_7);
+ d1 = highbd_convolve6_8_s32_s16(s1, s2, s3, s4, s5, s6, y_filter_0_7);
+ d2 = highbd_convolve6_8_s32_s16(s2, s3, s4, s5, s6, s7, y_filter_0_7);
+ d3 = highbd_convolve6_8_s32_s16(s3, s4, s5, s6, s7, s8, y_filter_0_7);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ if (h == 2) {
+ store_u16_8x2(d, dst_stride, d0, d1);
+ } else {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w > 0);
+ }
+}
+
static INLINE void highbd_convolve_y_sr_12tap_neon(
const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
int w, int h, const int16_t *y_filter_ptr, int bd) {
@@ -197,7 +322,6 @@
const InterpFilterParams *filter_params_y,
const int subpel_y_qn, int bd) {
const int y_filter_taps = get_filter_tap(filter_params_y, subpel_y_qn);
-
const int vert_offset = filter_params_y->taps / 2 - 1;
const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
@@ -211,6 +335,11 @@
y_filter_ptr, bd);
return;
}
+ if (y_filter_taps < 8) {
+ highbd_convolve_y_sr_6tap_neon(src, src_stride, dst, dst_stride, w, h,
+ y_filter_ptr, bd);
+ return;
+ }
const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
@@ -322,10 +451,10 @@
d2 = vminq_u16(d2, max);
d3 = vminq_u16(d3, max);
- if (h != 2) {
- store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
- } else {
+ if (h == 2) {
store_u16_8x2(d, dst_stride, d0, d1);
+ } else {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
}
s0 = s4;
diff --git a/av1/common/arm/highbd_convolve_neon.h b/av1/common/arm/highbd_convolve_neon.h
index 680c46e..1e2b28c 100644
--- a/av1/common/arm/highbd_convolve_neon.h
+++ b/av1/common/arm/highbd_convolve_neon.h
@@ -13,6 +13,66 @@
#include <arm_neon.h>
+static INLINE int32x4_t highbd_convolve6_4_s32(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ int32x4_t sum = vmull_lane_s16(s0, y_filter_lo, 1);
+ sum = vmlal_lane_s16(sum, s1, y_filter_lo, 2);
+ sum = vmlal_lane_s16(sum, s2, y_filter_lo, 3);
+ sum = vmlal_lane_s16(sum, s3, y_filter_hi, 0);
+ sum = vmlal_lane_s16(sum, s4, y_filter_hi, 1);
+ sum = vmlal_lane_s16(sum, s5, y_filter_hi, 2);
+
+ return sum;
+}
+
+static INLINE uint16x4_t highbd_convolve6_4_s32_s16(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x8_t y_filter) {
+ int32x4_t sum = highbd_convolve6_4_s32(s0, s1, s2, s3, s4, s5, y_filter);
+
+ return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
+}
+
+static INLINE void highbd_convolve6_8_s32(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter, int32x4_t *sum0, int32x4_t *sum1) {
+ const int16x4_t y_filter_lo = vget_low_s16(y_filter);
+ const int16x4_t y_filter_hi = vget_high_s16(y_filter);
+
+ *sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter_lo, 1);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s1), y_filter_lo, 2);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s2), y_filter_lo, 3);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s3), y_filter_hi, 0);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s4), y_filter_hi, 1);
+ *sum0 = vmlal_lane_s16(*sum0, vget_low_s16(s5), y_filter_hi, 2);
+
+ *sum1 = vmull_lane_s16(vget_high_s16(s0), y_filter_lo, 1);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s1), y_filter_lo, 2);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s2), y_filter_lo, 3);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s3), y_filter_hi, 0);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s4), y_filter_hi, 1);
+ *sum1 = vmlal_lane_s16(*sum1, vget_high_s16(s5), y_filter_hi, 2);
+}
+
+static INLINE uint16x8_t highbd_convolve6_8_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t y_filter) {
+ int32x4_t sum0;
+ int32x4_t sum1;
+ highbd_convolve6_8_s32(s0, s1, s2, s3, s4, s5, y_filter, &sum0, &sum1);
+
+ return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS),
+ vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
+}
+
static INLINE int32x4_t highbd_convolve8_4_s32(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
