[NEON] av1_highbd_convolve_2d_sr_neon() implementation
About 4x faster than C version.
Change-Id: I7e171d5db6a5df9571cc18eda281c7cd30a996be
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h
index 59de63c..58f5226 100644
--- a/aom_dsp/arm/mem_neon.h
+++ b/aom_dsp/arm/mem_neon.h
@@ -194,6 +194,13 @@
*s6 = vld1_u16(s);
}
+static INLINE void load_s16_8x2(const int16_t *s, const ptrdiff_t p,
+ int16x8_t *const s0, int16x8_t *const s1) {
+ *s0 = vld1q_s16(s);
+ s += p;
+ *s1 = vld1q_s16(s);
+}
+
static INLINE void load_u16_8x2(const uint16_t *s, const ptrdiff_t p,
uint16x8_t *const s0, uint16x8_t *const s1) {
*s0 = vld1q_u16(s);
diff --git a/av1/common/arm/highbd_convolve_neon.c b/av1/common/arm/highbd_convolve_neon.c
index 26b7df8..5a4b9c8 100644
--- a/av1/common/arm/highbd_convolve_neon.c
+++ b/av1/common/arm/highbd_convolve_neon.c
@@ -30,28 +30,17 @@
const int32x4_t zero_s32 = vdupq_n_s32(0);
if (w <= 4) {
- uint16x4_t t0, t1, t2, t3, t4, t5, t6, t7, t8;
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
uint16x4_t d0, d1, d2, d3;
uint16x8_t d01, d23;
-
- const uint16_t *s = src_ptr + src_stride;
+ const int16_t *s = (const int16_t *)(src_ptr + src_stride);
uint16_t *d = dst_ptr;
- load_u16_4x5(s, src_stride, &t0, &t1, &t2, &t3, &t4);
- s0 = vreinterpret_s16_u16(t0);
- s1 = vreinterpret_s16_u16(t1);
- s2 = vreinterpret_s16_u16(t2);
- s3 = vreinterpret_s16_u16(t3);
- s4 = vreinterpret_s16_u16(t4);
+ load_s16_4x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
s += 5 * src_stride;
do {
- load_u16_4x4(s, src_stride, &t5, &t6, &t7, &t8);
- s5 = vreinterpret_s16_u16(t5);
- s6 = vreinterpret_s16_u16(t6);
- s7 = vreinterpret_s16_u16(t7);
- s8 = vreinterpret_s16_u16(t8);
+ load_s16_4x4(s, src_stride, &s5, &s6, &s7, &s8);
d0 = highbd_convolve6_4_s32_s16(s0, s1, s2, s3, s4, s5, y_filter_0_7,
zero_s32);
@@ -95,29 +84,19 @@
} while (h > 0);
} else {
// if width is a multiple of 8 & height is a multiple of 4
- uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8;
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
uint16x8_t d0, d1, d2, d3;
do {
int height = h;
- const uint16_t *s = src_ptr + src_stride;
+ const int16_t *s = (const int16_t *)(src_ptr + src_stride);
uint16_t *d = dst_ptr;
- load_u16_8x5(s, src_stride, &t0, &t1, &t2, &t3, &t4);
- s0 = vreinterpretq_s16_u16(t0);
- s1 = vreinterpretq_s16_u16(t1);
- s2 = vreinterpretq_s16_u16(t2);
- s3 = vreinterpretq_s16_u16(t3);
- s4 = vreinterpretq_s16_u16(t4);
+ load_s16_8x5(s, src_stride, &s0, &s1, &s2, &s3, &s4);
s += 5 * src_stride;
do {
- load_u16_8x4(s, src_stride, &t5, &t6, &t7, &t8);
- s5 = vreinterpretq_s16_u16(t5);
- s6 = vreinterpretq_s16_u16(t6);
- s7 = vreinterpretq_s16_u16(t7);
- s8 = vreinterpretq_s16_u16(t8);
+ load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8);
d0 = highbd_convolve6_8_s32_s16(s0, s1, s2, s3, s4, s5, y_filter_0_7,
zero_s32);
@@ -164,31 +143,18 @@
const int32x4_t zero_s32 = vdupq_n_s32(0);
if (w <= 4) {
- uint16x4_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10;
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
uint16x4_t d0, d1, d2, d3;
uint16x8_t d01, d23;
- const uint16_t *s = src_ptr;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_u16_4x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
- s0 = vreinterpret_s16_u16(t0);
- s1 = vreinterpret_s16_u16(t1);
- s2 = vreinterpret_s16_u16(t2);
- s3 = vreinterpret_s16_u16(t3);
- s4 = vreinterpret_s16_u16(t4);
- s5 = vreinterpret_s16_u16(t5);
- s6 = vreinterpret_s16_u16(t6);
-
+ load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
s += 7 * src_stride;
do {
- load_u16_4x4(s, src_stride, &t7, &t8, &t9, &t10);
- s7 = vreinterpret_s16_u16(t7);
- s8 = vreinterpret_s16_u16(t8);
- s9 = vreinterpret_s16_u16(t9);
- s10 = vreinterpret_s16_u16(t10);
+ load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
d0 = highbd_convolve8_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, y_filter,
zero_s32);
@@ -233,32 +199,18 @@
h -= 4;
} while (h > 0);
} else {
- int height;
- uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10;
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
uint16x8_t d0, d1, d2, d3;
do {
- const uint16_t *s = src_ptr;
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_u16_8x7(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6);
- s0 = vreinterpretq_s16_u16(t0);
- s1 = vreinterpretq_s16_u16(t1);
- s2 = vreinterpretq_s16_u16(t2);
- s3 = vreinterpretq_s16_u16(t3);
- s4 = vreinterpretq_s16_u16(t4);
- s5 = vreinterpretq_s16_u16(t5);
- s6 = vreinterpretq_s16_u16(t6);
-
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
s += 7 * src_stride;
- height = h;
do {
- load_u16_8x4(s, src_stride, &t7, &t8, &t9, &t10);
- s7 = vreinterpretq_s16_u16(t7);
- s8 = vreinterpretq_s16_u16(t8);
- s9 = vreinterpretq_s16_u16(t9);
- s10 = vreinterpretq_s16_u16(t10);
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
d0 = highbd_convolve8_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
y_filter, zero_s32);
@@ -307,36 +259,19 @@
const int32x4_t zero_s32 = vdupq_n_s32(0);
if (w <= 4) {
- uint16x4_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14;
int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
uint16x4_t d0, d1, d2, d3;
uint16x8_t d01, d23;
- const uint16_t *s = src_ptr;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_u16_4x11(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, &t8,
- &t9, &t10);
- s0 = vreinterpret_s16_u16(t0);
- s1 = vreinterpret_s16_u16(t1);
- s2 = vreinterpret_s16_u16(t2);
- s3 = vreinterpret_s16_u16(t3);
- s4 = vreinterpret_s16_u16(t4);
- s5 = vreinterpret_s16_u16(t5);
- s6 = vreinterpret_s16_u16(t6);
- s7 = vreinterpret_s16_u16(t7);
- s8 = vreinterpret_s16_u16(t8);
- s9 = vreinterpret_s16_u16(t9);
- s10 = vreinterpret_s16_u16(t10);
-
+ load_s16_4x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+ &s9, &s10);
s += 11 * src_stride;
do {
- load_u16_4x4(s, src_stride, &t11, &t12, &t13, &t14);
- s11 = vreinterpret_s16_u16(t11);
- s12 = vreinterpret_s16_u16(t12);
- s13 = vreinterpret_s16_u16(t13);
- s14 = vreinterpret_s16_u16(t14);
+ load_s16_4x4(s, src_stride, &s11, &s12, &s13, &s14);
d0 = highbd_convolve12_y_4_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9,
s10, s11, y_filter_0_7, y_filter_8_11,
@@ -389,37 +324,20 @@
h -= 4;
} while (h > 0);
} else {
- uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14;
uint16x8_t d0, d1, d2, d3;
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
do {
- const uint16_t *s = src_ptr;
- uint16_t *d = dst_ptr;
int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
- load_u16_8x11(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, &t8,
- &t9, &t10);
- s0 = vreinterpretq_s16_u16(t0);
- s1 = vreinterpretq_s16_u16(t1);
- s2 = vreinterpretq_s16_u16(t2);
- s3 = vreinterpretq_s16_u16(t3);
- s4 = vreinterpretq_s16_u16(t4);
- s5 = vreinterpretq_s16_u16(t5);
- s6 = vreinterpretq_s16_u16(t6);
- s7 = vreinterpretq_s16_u16(t7);
- s8 = vreinterpretq_s16_u16(t8);
- s9 = vreinterpretq_s16_u16(t9);
- s10 = vreinterpretq_s16_u16(t10);
-
+ load_s16_8x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+ &s9, &s10);
s += 11 * src_stride;
do {
- load_u16_8x4(s, src_stride, &t11, &t12, &t13, &t14);
- s11 = vreinterpretq_s16_u16(t11);
- s12 = vreinterpretq_s16_u16(t12);
- s13 = vreinterpretq_s16_u16(t13);
- s14 = vreinterpretq_s16_u16(t14);
+ load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
d0 = highbd_convolve12_y_8_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7, s8,
s9, s10, s11, y_filter_0_7,
@@ -506,21 +424,16 @@
const int32x4_t zero_s32 = vdupq_n_s32(0);
if (w <= 4) {
- uint16x8_t t0, t1, t2, t3;
int16x8_t s0, s1, s2, s3;
uint16x4_t d0, d1;
uint16x8_t d01;
- const uint16_t *s = src_ptr;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
do {
- load_u16_8x2(s, src_stride, &t0, &t2);
- load_u16_8x2(s + 8, src_stride, &t1, &t3);
- s0 = vreinterpretq_s16_u16(t0);
- s1 = vreinterpretq_s16_u16(t1);
- s2 = vreinterpretq_s16_u16(t2);
- s3 = vreinterpretq_s16_u16(t3);
+ load_s16_8x2(s, src_stride, &s0, &s2);
+ load_s16_8x2(s + 8, src_stride, &s1, &s3);
d0 = highbd_convolve8_horiz4_s32_s16(s0, s1, x_filter, shift_s32,
zero_s32);
@@ -545,26 +458,18 @@
} while (h > 0);
} else {
int height = h;
- uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7;
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
uint16x8_t d0, d1, d2, d3;
do {
int width = w;
- const uint16_t *s = src_ptr;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_u16_8x4(s, src_stride, &t0, &t2, &t4, &t6);
- s0 = vreinterpretq_s16_u16(t0);
- s2 = vreinterpretq_s16_u16(t2);
- s4 = vreinterpretq_s16_u16(t4);
- s6 = vreinterpretq_s16_u16(t6);
+ load_s16_8x4(s, src_stride, &s0, &s2, &s4, &s6);
s += 8;
+
do {
- load_u16_8x4(s, src_stride, &t1, &t3, &t5, &t7);
- s1 = vreinterpretq_s16_u16(t1);
- s3 = vreinterpretq_s16_u16(t3);
- s5 = vreinterpretq_s16_u16(t5);
- s7 = vreinterpretq_s16_u16(t7);
+ load_s16_8x4(s, src_stride, &s1, &s3, &s5, &s7);
d0 = highbd_convolve8_horiz8_s32_s16(s0, s1, x_filter, shift_s32,
zero_s32);
@@ -619,21 +524,16 @@
const int32x4_t zero_s32 = vdupq_n_s32(0);
if (w <= 4) {
- uint16x8_t t0, t1, t2, t3;
int16x8_t s0, s1, s2, s3;
uint16x4_t d0, d1;
uint16x8_t d01;
- const uint16_t *s = src_ptr;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
do {
- load_u16_8x2(s, src_stride, &t0, &t2);
- load_u16_8x2(s + 8, src_stride, &t1, &t3);
- s0 = vreinterpretq_s16_u16(t0);
- s1 = vreinterpretq_s16_u16(t1);
- s2 = vreinterpretq_s16_u16(t2);
- s3 = vreinterpretq_s16_u16(t3);
+ load_s16_8x2(s, src_stride, &s0, &s2);
+ load_s16_8x2(s + 8, src_stride, &s1, &s3);
d0 = highbd_convolve12_horiz4_s32_s16(s0, s1, x_filter_0_7, x_filter_8_11,
shift_s32, zero_s32);
@@ -658,31 +558,19 @@
} while (h > 0);
} else {
int height = h;
- uint16x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11;
int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11;
uint16x8_t d0, d1, d2, d3;
do {
int width = w;
- const uint16_t *s = src_ptr;
+ const int16_t *s = (const int16_t *)src_ptr;
uint16_t *d = dst_ptr;
- load_u16_8x4(s, src_stride, &t0, &t3, &t6, &t9);
- s0 = vreinterpretq_s16_u16(t0);
- s3 = vreinterpretq_s16_u16(t3);
- s6 = vreinterpretq_s16_u16(t6);
- s9 = vreinterpretq_s16_u16(t9);
+ load_s16_8x4(s, src_stride, &s0, &s3, &s6, &s9);
s += 8;
+
do {
- load_u16_8x4(s, src_stride, &t1, &t4, &t7, &t10);
- load_u16_8x4(s + 8, src_stride, &t2, &t5, &t8, &t11);
- s1 = vreinterpretq_s16_u16(t1);
- s2 = vreinterpretq_s16_u16(t2);
- s4 = vreinterpretq_s16_u16(t4);
- s5 = vreinterpretq_s16_u16(t5);
- s7 = vreinterpretq_s16_u16(t7);
- s8 = vreinterpretq_s16_u16(t8);
- s10 = vreinterpretq_s16_u16(t10);
- s11 = vreinterpretq_s16_u16(t11);
+ load_s16_8x4(s, src_stride, &s1, &s4, &s7, &s10);
+ load_s16_8x4(s + 8, src_stride, &s2, &s5, &s8, &s11);
d0 = highbd_convolve12_horiz8_s32_s16(
s0, s1, s2, x_filter_0_7, x_filter_8_11, shift_s32, zero_s32);
@@ -749,3 +637,490 @@
highbd_convolve_x_sr_8tap_neon(src, src_stride, dst, dst_stride, w, h,
x_filter_ptr, conv_params, bd);
}
+
+static INLINE void highbd_convolve_2d_y_sr_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
+ int bd, const int offset, const int correction) {
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ const int16x8_t y_filter = vld1q_s16(y_filter_ptr);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+ const int round1_shift = conv_params->round_1;
+ const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_shift);
+ const int32x4_t correction_s32 = vdupq_n_s32(correction);
+
+ if (w <= 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ uint16x4_t d0, d1, d2, d3;
+ uint16x8_t d01, d23;
+
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ load_s16_4x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ load_s16_4x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ d0 = highbd_convolve8_4_rs_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, round1_shift_s32, offset_s32,
+ correction_s32);
+ d1 = highbd_convolve8_4_rs_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, round1_shift_s32, offset_s32,
+ correction_s32);
+ d2 = highbd_convolve8_4_rs_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, round1_shift_s32, offset_s32,
+ correction_s32);
+ d3 = highbd_convolve8_4_rs_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, round1_shift_s32, offset_s32,
+ correction_s32);
+
+ d01 = vcombine_u16(d0, d1);
+ d23 = vcombine_u16(d2, d3);
+
+ d01 = vminq_u16(d01, max);
+ d23 = vminq_u16(d23, max);
+
+ if (w == 2) {
+ store_u16q_2x1(d + 0 * dst_stride, d01, 0);
+ store_u16q_2x1(d + 1 * dst_stride, d01, 2);
+ if (h != 2) {
+ store_u16q_2x1(d + 2 * dst_stride, d23, 0);
+ store_u16q_2x1(d + 3 * dst_stride, d23, 2);
+ }
+ } else {
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ if (h != 2) {
+ vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
+ vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
+ }
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
+ uint16x8_t d0, d1, d2, d3;
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ load_s16_8x7(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6);
+ s += 7 * src_stride;
+
+ do {
+ load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10);
+
+ d0 = highbd_convolve8_8_rs_s32_s16(s0, s1, s2, s3, s4, s5, s6, s7,
+ y_filter, round1_shift_s32,
+ offset_s32, correction_s32);
+ d1 = highbd_convolve8_8_rs_s32_s16(s1, s2, s3, s4, s5, s6, s7, s8,
+ y_filter, round1_shift_s32,
+ offset_s32, correction_s32);
+ d2 = highbd_convolve8_8_rs_s32_s16(s2, s3, s4, s5, s6, s7, s8, s9,
+ y_filter, round1_shift_s32,
+ offset_s32, correction_s32);
+ d3 = highbd_convolve8_8_rs_s32_s16(s3, s4, s5, s6, s7, s8, s9, s10,
+ y_filter, round1_shift_s32,
+ offset_s32, correction_s32);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ if (h == 2) {
+ store_u16_8x2(d, dst_stride, d0, d1);
+ } else {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w > 0);
+ }
+}
+
+static INLINE void highbd_convolve_2d_y_sr_12tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *y_filter_ptr, ConvolveParams *conv_params,
+ const int bd, const int offset, const int correction) {
+ const uint16x8_t max = vdupq_n_u16((1 << bd) - 1);
+ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr);
+ const int16x4_t y_filter_8_11 = vld1_s16(y_filter_ptr + 8);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+ const int round1_shift = conv_params->round_1;
+ const int32x4_t round1_shift_s32 = vdupq_n_s32(-round1_shift);
+ const int32x4_t correction_s32 = vdupq_n_s32(correction);
+
+ if (w <= 4) {
+ int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
+ uint16x4_t d0, d1, d2, d3;
+ uint16x8_t d01, d23;
+
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ load_s16_4x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+ &s9, &s10);
+ s += 11 * src_stride;
+
+ do {
+ load_s16_4x4(s, src_stride, &s11, &s12, &s13, &s14);
+
+ d0 = highbd_convolve12_y_4_rs_s32_s16(
+ s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ d1 = highbd_convolve12_y_4_rs_s32_s16(
+ s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ d2 = highbd_convolve12_y_4_rs_s32_s16(
+ s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ d3 = highbd_convolve12_y_4_rs_s32_s16(
+ s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+
+ d01 = vcombine_u16(d0, d1);
+ d23 = vcombine_u16(d2, d3);
+
+ d01 = vminq_u16(d01, max);
+ d23 = vminq_u16(d23, max);
+
+ if (w == 2) {
+ store_u16q_2x1(d + 0 * dst_stride, d01, 0);
+ store_u16q_2x1(d + 1 * dst_stride, d01, 2);
+ if (h != 2) {
+ store_u16q_2x1(d + 2 * dst_stride, d23, 0);
+ store_u16q_2x1(d + 3 * dst_stride, d23, 2);
+ }
+ } else {
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ if (h != 2) {
+ vst1_u16(d + 2 * dst_stride, vget_low_u16(d23));
+ vst1_u16(d + 3 * dst_stride, vget_high_u16(d23));
+ }
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s7 = s11;
+ s8 = s12;
+ s9 = s13;
+ s10 = s14;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ h -= 4;
+ } while (h > 0);
+ } else {
+ uint16x8_t d0, d1, d2, d3;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14;
+
+ do {
+ int height = h;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ load_s16_8x11(s, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8,
+ &s9, &s10);
+ s += 11 * src_stride;
+
+ do {
+ load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14);
+
+ d0 = highbd_convolve12_y_8_rs_s32_s16(
+ s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ d1 = highbd_convolve12_y_8_rs_s32_s16(
+ s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ d2 = highbd_convolve12_y_8_rs_s32_s16(
+ s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+ d3 = highbd_convolve12_y_8_rs_s32_s16(
+ s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, y_filter_0_7,
+ y_filter_8_11, round1_shift_s32, offset_s32, correction_s32);
+
+ d0 = vminq_u16(d0, max);
+ d1 = vminq_u16(d1, max);
+ d2 = vminq_u16(d2, max);
+ d3 = vminq_u16(d3, max);
+
+ if (h == 2) {
+ store_u16_8x2(d, dst_stride, d0, d1);
+ } else {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+ }
+
+ s0 = s4;
+ s1 = s5;
+ s2 = s6;
+ s3 = s7;
+ s4 = s8;
+ s5 = s9;
+ s6 = s10;
+ s7 = s11;
+ s8 = s12;
+ s9 = s13;
+ s10 = s14;
+ s += 4 * src_stride;
+ d += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+
+ src_ptr += 8;
+ dst_ptr += 8;
+ w -= 8;
+ } while (w > 0);
+ }
+}
+
+static INLINE void highbd_convolve_2d_x_sr_8tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
+ const int offset) {
+ const int16x8_t x_filter = vld1q_s16(x_filter_ptr);
+ const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+
+ if (w <= 4) {
+ int16x8_t s0, s1, s2, s3;
+ uint16x4_t d0, d1;
+ uint16x8_t d01;
+
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ load_s16_8x2(s, src_stride, &s0, &s2);
+ load_s16_8x2(s + 8, src_stride, &s1, &s3);
+
+ d0 = highbd_convolve8_horiz4_s32_s16(s0, s1, x_filter, shift_s32,
+ offset_s32);
+ d1 = highbd_convolve8_horiz4_s32_s16(s2, s3, x_filter, shift_s32,
+ offset_s32);
+
+ d01 = vcombine_u16(d0, d1);
+
+ if (w == 2) {
+ store_u16q_2x1(d + 0 * dst_stride, d01, 0);
+ store_u16q_2x1(d + 1 * dst_stride, d01, 2);
+ } else {
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ }
+
+ s += 2 * src_stride;
+ d += 2 * dst_stride;
+ h -= 2;
+ } while (h > 0);
+ } else {
+ int height = h;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
+ uint16x8_t d0, d1, d2, d3;
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ load_s16_8x4(s, src_stride, &s0, &s2, &s4, &s6);
+ s += 8;
+
+ do {
+ load_s16_8x4(s, src_stride, &s1, &s3, &s5, &s7);
+
+ d0 = highbd_convolve8_horiz8_s32_s16(s0, s1, x_filter, shift_s32,
+ offset_s32);
+ d1 = highbd_convolve8_horiz8_s32_s16(s2, s3, x_filter, shift_s32,
+ offset_s32);
+ d2 = highbd_convolve8_horiz8_s32_s16(s4, s5, x_filter, shift_s32,
+ offset_s32);
+ d3 = highbd_convolve8_horiz8_s32_s16(s6, s7, x_filter, shift_s32,
+ offset_s32);
+
+ if (h == 2) {
+ store_u16_8x2(d, dst_stride, d0, d1);
+ } else {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+ }
+
+ s0 = s1;
+ s2 = s3;
+ s4 = s5;
+ s6 = s7;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+ }
+}
+
+static INLINE void highbd_convolve_2d_x_sr_12tap_neon(
+ const uint16_t *src_ptr, int src_stride, uint16_t *dst_ptr, int dst_stride,
+ int w, int h, const int16_t *x_filter_ptr, ConvolveParams *conv_params,
+ const int offset) {
+ const int32x4_t shift_s32 = vdupq_n_s32(-conv_params->round_0);
+ const int16x8_t x_filter_0_7 = vld1q_s16(x_filter_ptr);
+ const int16x4_t x_filter_8_11 = vld1_s16(x_filter_ptr + 8);
+ const int32x4_t offset_s32 = vdupq_n_s32(offset);
+
+ if (w <= 4) {
+ int16x8_t s0, s1, s2, s3;
+ uint16x4_t d0, d1;
+ uint16x8_t d01;
+
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ do {
+ load_s16_8x2(s, src_stride, &s0, &s2);
+ load_s16_8x2(s + 8, src_stride, &s1, &s3);
+
+ d0 = highbd_convolve12_horiz4_s32_s16(s0, s1, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
+ d1 = highbd_convolve12_horiz4_s32_s16(s2, s3, x_filter_0_7, x_filter_8_11,
+ shift_s32, offset_s32);
+
+ d01 = vcombine_u16(d0, d1);
+
+ if (w == 2) {
+ store_u16q_2x1(d + 0 * dst_stride, d01, 0);
+ store_u16q_2x1(d + 1 * dst_stride, d01, 2);
+ } else {
+ vst1_u16(d + 0 * dst_stride, vget_low_u16(d01));
+ vst1_u16(d + 1 * dst_stride, vget_high_u16(d01));
+ }
+
+ s += 2 * src_stride;
+ d += 2 * dst_stride;
+ h -= 2;
+ } while (h > 0);
+ } else {
+ int height = h;
+ int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11;
+ uint16x8_t d0, d1, d2, d3;
+ do {
+ int width = w;
+ const int16_t *s = (const int16_t *)src_ptr;
+ uint16_t *d = dst_ptr;
+
+ load_s16_8x4(s, src_stride, &s0, &s3, &s6, &s9);
+ s += 8;
+
+ do {
+ load_s16_8x4(s, src_stride, &s1, &s4, &s7, &s10);
+ load_s16_8x4(s + 8, src_stride, &s2, &s5, &s8, &s11);
+
+ d0 = highbd_convolve12_horiz8_s32_s16(
+ s0, s1, s2, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ d1 = highbd_convolve12_horiz8_s32_s16(
+ s3, s4, s5, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ d2 = highbd_convolve12_horiz8_s32_s16(
+ s6, s7, s8, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+ d3 = highbd_convolve12_horiz8_s32_s16(
+ s9, s10, s11, x_filter_0_7, x_filter_8_11, shift_s32, offset_s32);
+
+ if (h == 2) {
+ store_u16_8x2(d, dst_stride, d0, d1);
+ } else {
+ store_u16_8x4(d, dst_stride, d0, d1, d2, d3);
+ }
+
+ s0 = s1;
+ s1 = s2;
+ s3 = s4;
+ s4 = s5;
+ s6 = s7;
+ s7 = s8;
+ s9 = s10;
+ s10 = s11;
+ s += 8;
+ d += 8;
+ width -= 8;
+ } while (width > 0);
+ src_ptr += 4 * src_stride;
+ dst_ptr += 4 * dst_stride;
+ height -= 4;
+ } while (height > 0);
+ }
+}
+
+void av1_highbd_convolve_2d_sr_neon(const uint16_t *src, int src_stride,
+ uint16_t *dst, int dst_stride, int w, int h,
+ const InterpFilterParams *filter_params_x,
+ const InterpFilterParams *filter_params_y,
+ const int subpel_x_qn,
+ const int subpel_y_qn,
+ ConvolveParams *conv_params, int bd) {
+ DECLARE_ALIGNED(16, uint16_t,
+ im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]);
+ const int im_h = h + filter_params_y->taps - 1;
+ const int im_stride = MAX_SB_SIZE;
+ const int vert_offset = filter_params_y->taps / 2 - 1;
+ const int horiz_offset = filter_params_x->taps / 2 - 1;
+ const int x_offset_bits = (1 << (bd + FILTER_BITS - 1));
+ const int y_offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
+ const int y_offset_initial = (1 << y_offset_bits);
+ const int y_offset_correction_s32 =
+ ((1 << (y_offset_bits - conv_params->round_1)) +
+ (1 << (y_offset_bits - conv_params->round_1 - 1)));
+
+ const uint16_t *src_ptr = src - vert_offset * src_stride - horiz_offset;
+
+ const int16_t *x_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_x, subpel_x_qn & SUBPEL_MASK);
+ const int16_t *y_filter_ptr = av1_get_interp_filter_subpel_kernel(
+ filter_params_y, subpel_y_qn & SUBPEL_MASK);
+
+ if (filter_params_x->taps > 8) {
+ highbd_convolve_2d_x_sr_12tap_neon(src_ptr, src_stride, im_block, im_stride,
+ w, im_h, x_filter_ptr, conv_params,
+ x_offset_bits);
+
+ highbd_convolve_2d_y_sr_12tap_neon(
+ im_block, im_stride, dst, dst_stride, w, h, y_filter_ptr, conv_params,
+ bd, y_offset_initial, y_offset_correction_s32);
+ } else {
+ highbd_convolve_2d_x_sr_8tap_neon(src_ptr, src_stride, im_block, im_stride,
+ w, im_h, x_filter_ptr, conv_params,
+ x_offset_bits);
+
+ highbd_convolve_2d_y_sr_8tap_neon(
+ im_block, im_stride, dst, dst_stride, w, h, y_filter_ptr, conv_params,
+ bd, y_offset_initial, y_offset_correction_s32);
+ }
+}
diff --git a/av1/common/arm/highbd_convolve_neon.h b/av1/common/arm/highbd_convolve_neon.h
index fde881c..7934463 100644
--- a/av1/common/arm/highbd_convolve_neon.h
+++ b/av1/common/arm/highbd_convolve_neon.h
@@ -108,6 +108,20 @@
return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
}
+// Like above but also perform round shifting and subtract correction term
+static INLINE uint16x4_t highbd_convolve8_4_rs_s32_s16(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x8_t y_filter,
+ const int32x4_t round_shift, const int32x4_t offset,
+ const int32x4_t correction) {
+ int32x4_t sum =
+ highbd_convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, offset);
+
+ sum = vsubq_s32(vqrshlq_s32(sum, round_shift), correction);
+ return vqmovun_s32(sum);
+}
+
static INLINE void highbd_convolve8_8_s32(
const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
@@ -149,6 +163,24 @@
vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
}
+// Like above but also perform round shifting and subtract correction term
+static INLINE uint16x8_t highbd_convolve8_8_rs_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t y_filter,
+ const int32x4_t round_shift, const int32x4_t offset,
+ const int32x4_t correction) {
+ int32x4_t sum0;
+ int32x4_t sum1;
+ highbd_convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, offset,
+ &sum0, &sum1);
+
+ sum0 = vsubq_s32(vqrshlq_s32(sum0, round_shift), correction);
+ sum1 = vsubq_s32(vqrshlq_s32(sum1, round_shift), correction);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
static INLINE int32x4_t highbd_convolve12_y_4_s32(
const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
@@ -189,6 +221,23 @@
return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS);
}
+// Like above but also perform round shifting and subtract correction term
+static INLINE uint16x4_t highbd_convolve12_y_4_rs_s32_s16(
+ const int16x4_t s0, const int16x4_t s1, const int16x4_t s2,
+ const int16x4_t s3, const int16x4_t s4, const int16x4_t s5,
+ const int16x4_t s6, const int16x4_t s7, const int16x4_t s8,
+ const int16x4_t s9, const int16x4_t s10, const int16x4_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
+ const int32x4_t round_shift, const int32x4_t offset,
+ const int32x4_t correction) {
+ int32x4_t sum =
+ highbd_convolve12_y_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10,
+ s11, y_filter_0_7, y_filter_8_11, offset);
+
+ sum = vsubq_s32(vqrshlq_s32(sum, round_shift), correction);
+ return vqmovun_s32(sum);
+}
+
static INLINE void highbd_convolve12_y_8_s32(
const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
@@ -242,6 +291,26 @@
vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS));
}
+// Like above but also perform round shifting and subtract correction term
+static INLINE uint16x8_t highbd_convolve12_y_8_rs_s32_s16(
+ const int16x8_t s0, const int16x8_t s1, const int16x8_t s2,
+ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5,
+ const int16x8_t s6, const int16x8_t s7, const int16x8_t s8,
+ const int16x8_t s9, const int16x8_t s10, const int16x8_t s11,
+ const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11,
+ const int32x4_t round_shift, const int32x4_t offset,
+ const int32x4_t correction) {
+ int32x4_t sum0;
+ int32x4_t sum1;
+ highbd_convolve12_y_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11,
+ y_filter_0_7, y_filter_8_11, offset, &sum0, &sum1);
+
+ sum0 = vsubq_s32(vqrshlq_s32(sum0, round_shift), correction);
+ sum1 = vsubq_s32(vqrshlq_s32(sum1, round_shift), correction);
+
+ return vcombine_u16(vqmovun_s32(sum0), vqmovun_s32(sum1));
+}
+
static INLINE int32x4_t highbd_convolve8_horiz4_s32(
const int16x8_t s0, const int16x8_t s1, const int16x8_t x_filter_0_7,
const int32x4_t offset) {
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 176c6f7..4a38d7d 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -595,7 +595,7 @@
specialize qw/av1_highbd_dist_wtd_convolve_x sse4_1 avx2/;
specialize qw/av1_highbd_dist_wtd_convolve_y sse4_1 avx2/;
specialize qw/av1_highbd_dist_wtd_convolve_2d_copy sse4_1 avx2/;
- specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2/;
+ specialize qw/av1_highbd_convolve_2d_sr ssse3 avx2 neon/;
specialize qw/av1_highbd_convolve_x_sr ssse3 avx2 neon/;
specialize qw/av1_highbd_convolve_y_sr ssse3 avx2 neon/;
specialize qw/av1_highbd_convolve_2d_scale sse4_1/;
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc
index 1453d0d..964065c 100644
--- a/test/av1_convolve_test.cc
+++ b/test/av1_convolve_test.cc
@@ -1082,6 +1082,11 @@
BuildHighbdParams(av1_highbd_convolve_2d_sr_avx2));
#endif
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(NEON, AV1Convolve2DHighbdTest,
+ BuildHighbdParams(av1_highbd_convolve_2d_sr_neon));
+#endif
+
#endif // CONFIG_AV1_HIGHBITDEPTH
//////////////////////////
