Add Neon version of av1_wiener_convolve_add_src Block Scaling over C 8x4 ~5.33x 8x8 ~5.75x 32x8 ~6.76x 32x28 ~7.56x 64x56 ~7.50x 64x64 ~7.47x Change-Id: Ib143cf8b2a77b348136a198f6842d347f96df951
diff --git a/av1/av1.cmake b/av1/av1.cmake index 6f6436f..e2407e4 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -231,8 +231,10 @@ set(AOM_AV1_COMMON_INTRIN_NEON ${AOM_AV1_COMMON_INTRIN_NEON} "${AOM_ROOT}/av1/common/cdef_block_neon.c" "${AOM_ROOT}/av1/common/arm/convolve_neon.c" + "${AOM_ROOT}/av1/common/arm/convolve_neon.h" + "${AOM_ROOT}/av1/common/arm/mem_neon.h" "${AOM_ROOT}/av1/common/arm/transpose_neon.h" - "${AOM_ROOT}/av1/common/arm/mem_neon.h") + "${AOM_ROOT}/av1/common/arm/wiener_convolve_neon.c") set(AOM_AV1_COMMON_INTRIN_SSE2 ${AOM_AV1_COMMON_INTRIN_SSE2} "${AOM_ROOT}/av1/common/x86/convolve_2d_sse2.c")
diff --git a/av1/common/arm/convolve_neon.h b/av1/common/arm/convolve_neon.h new file mode 100644 index 0000000..2773cc0 --- /dev/null +++ b/av1/common/arm/convolve_neon.h
@@ -0,0 +1,158 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef AV1_COMMON_ARM_CONVOLVE_NEON_H_ +#define AV1_COMMON_ARM_CONVOLVE_NEON_H_ + +#include <arm_neon.h> + +#define HORIZ_EXTRA_ROWS ((SUBPEL_TAPS + 7) & ~0x07) + +static INLINE uint8x8_t wiener_convolve8_vert_4x8( + const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, + const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, + const int16x8_t s6, int16_t *filter_y, const int bd, + const int round1_bits) { + int16x8_t ss0, ss1, ss2; + int32x4_t sum0, sum1; + uint16x4_t tmp0, tmp1; + uint16x8_t tmp; + uint8x8_t res; + + const int32_t round_const = (1 << (bd + round1_bits - 1)); + const int32x4_t round_bits = vdupq_n_s32(-round1_bits); + const int32x4_t zero = vdupq_n_s32(0); + const int32x4_t round_vec = vdupq_n_s32(round_const); + + ss0 = vaddq_s16(s0, s6); + ss1 = vaddq_s16(s1, s5); + ss2 = vaddq_s16(s2, s4); + + sum0 = vmull_n_s16(vget_low_s16(ss0), filter_y[0]); + sum0 = vmlal_n_s16(sum0, vget_low_s16(ss1), filter_y[1]); + sum0 = vmlal_n_s16(sum0, vget_low_s16(ss2), filter_y[2]); + sum0 = vmlal_n_s16(sum0, vget_low_s16(s3), filter_y[3]); + + sum1 = vmull_n_s16(vget_high_s16(ss0), filter_y[0]); + sum1 = vmlal_n_s16(sum1, vget_high_s16(ss1), filter_y[1]); + sum1 = vmlal_n_s16(sum1, vget_high_s16(ss2), filter_y[2]); + sum1 = vmlal_n_s16(sum1, vget_high_s16(s3), filter_y[3]); + + sum0 = vsubq_s32(sum0, round_vec); + sum1 = vsubq_s32(sum1, round_vec); + + /* right shift & rounding */ + sum0 = vrshlq_s32(sum0, round_bits); + sum1 = vrshlq_s32(sum1, round_bits); + + sum0 = vmaxq_s32(sum0, zero); + sum1 = vmaxq_s32(sum1, zero); + + /* from int32x4_t to uint8x8_t */ + tmp0 = vqmovn_u32(vreinterpretq_u32_s32(sum0)); + tmp1 = vqmovn_u32(vreinterpretq_u32_s32(sum1)); + tmp = vcombine_u16(tmp0, tmp1); + res = vqmovn_u16(tmp); + + return res; +} + +static INLINE uint16x8_t wiener_convolve8_horiz_8x8( + const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, + const int16x8_t s3, int16_t *filter_x, const int bd, + const int round0_bits) { + int16x8_t sum; + uint16x8_t res; + int32x4_t sum_0, sum_1; + int32x4_t s3_0, s3_1; + const int32_t round_const_0 = (1 << (bd + FILTER_BITS - 1)); + const int32_t round_const_1 = (1 << ((bd) + 1 + FILTER_BITS - round0_bits)); + + /* for the purpose of right shift by { conv_params->round_0 } */ + const int32x4_t round_bits = vdupq_n_s32(-round0_bits); + + const int32x4_t round_vec_0 = vdupq_n_s32(round_const_0); + const int32x4_t round_vec_1 = vdupq_n_s32(round_const_1); + + sum = vmulq_n_s16(s0, filter_x[0]); + sum = vmlaq_n_s16(sum, s1, filter_x[1]); + sum = vmlaq_n_s16(sum, s2, filter_x[2]); + + /* sum from 16x8 to 2 32x4 registers */ + sum_0 = vmovl_s16(vget_low_s16(sum)); + sum_1 = vmovl_s16(vget_high_s16(sum)); + + /* s[3]*128 -- and filter coef max can be 128 + then max value possible = 128*128*255 exceeding 16 bit + */ + + s3_0 = vmull_n_s16(vget_low_s16(s3), filter_x[3]); + s3_1 = vmull_n_s16(vget_high_s16(s3), filter_x[3]); + sum_0 = vaddq_s32(sum_0, s3_0); + sum_1 = vaddq_s32(sum_1, s3_1); + + /* Add the constant value */ + sum_0 = vaddq_s32(sum_0, round_vec_0); + sum_1 = vaddq_s32(sum_1, round_vec_0); + + /* right shift & rounding & saturating */ + sum_0 = vqrshlq_s32(sum_0, round_bits); + sum_1 = vqrshlq_s32(sum_1, round_bits); + + /* Clipping to max value */ + sum_0 = vminq_s32(sum_0, round_vec_1); + sum_1 = vminq_s32(sum_1, round_vec_1); + + res = vcombine_u16(vqmovun_s32(sum_0), vqmovun_s32(sum_1)); + return res; +} + +static INLINE uint16x4_t wiener_convolve8_horiz_4x8( + const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, + const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, + const int16x4_t s6, int16_t *filter_x, const int bd, + const int round0_bits) { + uint16x4_t res; + int32x4_t sum_0, s3_0; + int16x4_t sum, temp0, temp1, temp2; + + const int32_t round_const_0 = (1 << (bd + FILTER_BITS - 1)); + const int32_t round_const_1 = (1 << ((bd) + 1 + FILTER_BITS - round0_bits)); + const int32x4_t round_bits = vdupq_n_s32(-round0_bits); + const int32x4_t zero = vdupq_n_s32(0); + const int32x4_t round_vec_0 = vdupq_n_s32(round_const_0); + const int32x4_t round_vec_1 = vdupq_n_s32(round_const_1); + + temp0 = vadd_s16(s0, s6); + temp1 = vadd_s16(s1, s5); + temp2 = vadd_s16(s2, s4); + + sum = vmul_n_s16(temp0, filter_x[0]); + sum = vmla_n_s16(sum, temp1, filter_x[1]); + sum = vmla_n_s16(sum, temp2, filter_x[2]); + sum_0 = vmovl_s16(sum); + + /* s[3]*128 -- and filter coff max can be 128. + then max value possible = 128*128*255 Therefore, 32 bits are required to + hold the result. + */ + s3_0 = vmull_n_s16(s3, filter_x[3]); + sum_0 = vaddq_s32(sum_0, s3_0); + + sum_0 = vaddq_s32(sum_0, round_vec_0); + sum_0 = vrshlq_s32(sum_0, round_bits); + + sum_0 = vmaxq_s32(sum_0, zero); + sum_0 = vminq_s32(sum_0, round_vec_1); + res = vqmovun_s32(sum_0); + return res; +} + +#endif // AV1_COMMON_ARM_CONVOLVE_NEON_H_
diff --git a/av1/common/arm/mem_neon.h b/av1/common/arm/mem_neon.h index 791f465..daff54e 100644 --- a/av1/common/arm/mem_neon.h +++ b/av1/common/arm/mem_neon.h
@@ -55,6 +55,18 @@ *s3 = vld1_u8(s); } +static INLINE void load_s16_8x8(const uint8_t *s, const ptrdiff_t p, + uint8x8_t *const s0, uint8x8_t *const s1, + uint8x8_t *const s2, uint8x8_t *const s3) { + *s0 = vld1_u8(s); + s += p; + *s1 = vld1_u8(s); + s += p; + *s2 = vld1_u8(s); + s += p; + *s3 = vld1_u8(s); +} + static INLINE void store_u8_8x8(uint8_t *s, ptrdiff_t p, const uint8x8_t s0, const uint8x8_t s1, const uint8x8_t s2, const uint8x8_t s3, const uint8x8_t s4, @@ -77,4 +89,38 @@ vst1_u8(s, s7); } +static INLINE void store_u16_8x8(uint16_t *s, ptrdiff_t dst_stride, + const uint16x8_t s0, const uint16x8_t s1, + const uint16x8_t s2, const uint16x8_t s3, + const uint16x8_t s4, const uint16x8_t s5, + const uint16x8_t s6, const uint16x8_t s7) { + vst1q_u16(s, s0); + s += dst_stride; + vst1q_u16(s, s1); + s += dst_stride; + vst1q_u16(s, s2); + s += dst_stride; + vst1q_u16(s, s3); + s += dst_stride; + vst1q_u16(s, s4); + s += dst_stride; + vst1q_u16(s, s5); + s += dst_stride; + vst1q_u16(s, s6); + s += dst_stride; + vst1q_u16(s, s7); +} + +static INLINE void store_u16_8x4(uint16_t *s, ptrdiff_t dst_stride, + const uint16x8_t s0, const uint16x8_t s1, + const uint16x8_t s2, const uint16x8_t s3) { + vst1q_u16(s, s0); + s += dst_stride; + vst1q_u16(s, s1); + s += dst_stride; + vst1q_u16(s, s2); + s += dst_stride; + vst1q_u16(s, s3); +} + #endif // AV1_COMMON_ARM_MEM_NEON_H_
diff --git a/av1/common/arm/transpose_neon.h b/av1/common/arm/transpose_neon.h index d12b96f..d8c7cc1 100644 --- a/av1/common/arm/transpose_neon.h +++ b/av1/common/arm/transpose_neon.h
@@ -184,4 +184,138 @@ *a2 = d1.val[0]; *a3 = d1.val[1]; } + +static INLINE void transpose_u16_4x8(uint16x4_t *a0, uint16x4_t *a1, + uint16x4_t *a2, uint16x4_t *a3, + uint16x4_t *a4, uint16x4_t *a5, + uint16x4_t *a6, uint16x4_t *a7, + uint16x8_t *o0, uint16x8_t *o1, + uint16x8_t *o2, uint16x8_t *o3) { + // Swap 16 bit elements. Goes from: + // a0: 00 01 02 03 + // a1: 10 11 12 13 + // a2: 20 21 22 23 + // a3: 30 31 32 33 + // a4: 40 41 42 43 + // a5: 50 51 52 53 + // a6: 60 61 62 63 + // a7: 70 71 72 73 + // to: + // b0.val[0]: 00 10 02 12 + // b0.val[1]: 01 11 03 13 + // b1.val[0]: 20 30 22 32 + // b1.val[1]: 21 31 23 33 + // b2.val[0]: 40 50 42 52 + // b2.val[1]: 41 51 43 53 + // b3.val[0]: 60 70 62 72 + // b3.val[1]: 61 71 63 73 + + uint16x4x2_t b0 = vtrn_u16(*a0, *a1); + uint16x4x2_t b1 = vtrn_u16(*a2, *a3); + uint16x4x2_t b2 = vtrn_u16(*a4, *a5); + uint16x4x2_t b3 = vtrn_u16(*a6, *a7); + + // Swap 32 bit elements resulting in: + // c0.val[0]: 00 10 20 30 + // c0.val[1]: 02 12 22 32 + // c1.val[0]: 01 11 21 31 + // c1.val[1]: 03 13 23 33 + // c2.val[0]: 40 50 60 70 + // c2.val[1]: 42 52 62 72 + // c3.val[0]: 41 51 61 71 + // c3.val[1]: 43 53 63 73 + + uint32x2x2_t c0 = vtrn_u32(vreinterpret_u32_u16(b0.val[0]), + vreinterpret_u32_u16(b1.val[0])); + uint32x2x2_t c1 = vtrn_u32(vreinterpret_u32_u16(b0.val[1]), + vreinterpret_u32_u16(b1.val[1])); + uint32x2x2_t c2 = vtrn_u32(vreinterpret_u32_u16(b2.val[0]), + vreinterpret_u32_u16(b3.val[0])); + uint32x2x2_t c3 = vtrn_u32(vreinterpret_u32_u16(b2.val[1]), + vreinterpret_u32_u16(b3.val[1])); + + // Swap 64 bit elements resulting in: + // o0: 00 10 20 30 40 50 60 70 + // o1: 01 11 21 31 41 51 61 71 + // o2: 02 12 22 32 42 52 62 72 + // o3: 03 13 23 33 43 53 63 73 + + *o0 = vcombine_u16(vreinterpret_u16_u32(c0.val[0]), + vreinterpret_u16_u32(c2.val[0])); + *o1 = vcombine_u16(vreinterpret_u16_u32(c1.val[0]), + vreinterpret_u16_u32(c3.val[0])); + *o2 = vcombine_u16(vreinterpret_u16_u32(c0.val[1]), + vreinterpret_u16_u32(c2.val[1])); + *o3 = vcombine_u16(vreinterpret_u16_u32(c1.val[1]), + vreinterpret_u16_u32(c3.val[1])); +} + +static INLINE void transpose_u16_8x8(uint16x8_t *a0, uint16x8_t *a1, + uint16x8_t *a2, uint16x8_t *a3, + uint16x8_t *a4, uint16x8_t *a5, + uint16x8_t *a6, uint16x8_t *a7) { + // Swap 16 bit elements. Goes from: + // a0: 00 01 02 03 04 05 06 07 + // a1: 10 11 12 13 14 15 16 17 + // a2: 20 21 22 23 24 25 26 27 + // a3: 30 31 32 33 34 35 36 37 + // a4: 40 41 42 43 44 45 46 47 + // a5: 50 51 52 53 54 55 56 57 + // a6: 60 61 62 63 64 65 66 67 + // a7: 70 71 72 73 74 75 76 77 + // to: + // b0.val[0]: 00 10 02 12 04 14 06 16 + // b0.val[1]: 01 11 03 13 05 15 07 17 + // b1.val[0]: 20 30 22 32 24 34 26 36 + // b1.val[1]: 21 31 23 33 25 35 27 37 + // b2.val[0]: 40 50 42 52 44 54 46 56 + // b2.val[1]: 41 51 43 53 45 55 47 57 + // b3.val[0]: 60 70 62 72 64 74 66 76 + // b3.val[1]: 61 71 63 73 65 75 67 77 + + const uint16x8x2_t b0 = vtrnq_u16(*a0, *a1); + const uint16x8x2_t b1 = vtrnq_u16(*a2, *a3); + const uint16x8x2_t b2 = vtrnq_u16(*a4, *a5); + const uint16x8x2_t b3 = vtrnq_u16(*a6, *a7); + + // Swap 32 bit elements resulting in: + // c0.val[0]: 00 10 20 30 04 14 24 34 + // c0.val[1]: 02 12 22 32 06 16 26 36 + // c1.val[0]: 01 11 21 31 05 15 25 35 + // c1.val[1]: 03 13 23 33 07 17 27 37 + // c2.val[0]: 40 50 60 70 44 54 64 74 + // c2.val[1]: 42 52 62 72 46 56 66 76 + // c3.val[0]: 41 51 61 71 45 55 65 75 + // c3.val[1]: 43 53 63 73 47 57 67 77 + + const uint32x4x2_t c0 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[0]), + vreinterpretq_u32_u16(b1.val[0])); + const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[1]), + vreinterpretq_u32_u16(b1.val[1])); + const uint32x4x2_t c2 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[0]), + vreinterpretq_u32_u16(b3.val[0])); + const uint32x4x2_t c3 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[1]), + vreinterpretq_u32_u16(b3.val[1])); + + *a0 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c0.val[0])), + vget_low_u16(vreinterpretq_u16_u32(c2.val[0]))); + *a4 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c0.val[0])), + vget_high_u16(vreinterpretq_u16_u32(c2.val[0]))); + + *a2 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c0.val[1])), + vget_low_u16(vreinterpretq_u16_u32(c2.val[1]))); + *a6 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c0.val[1])), + vget_high_u16(vreinterpretq_u16_u32(c2.val[1]))); + + *a1 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c1.val[0])), + vget_low_u16(vreinterpretq_u16_u32(c3.val[0]))); + *a5 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c1.val[0])), + vget_high_u16(vreinterpretq_u16_u32(c3.val[0]))); + + *a3 = vcombine_u16(vget_low_u16(vreinterpretq_u16_u32(c1.val[1])), + vget_low_u16(vreinterpretq_u16_u32(c3.val[1]))); + *a7 = vcombine_u16(vget_high_u16(vreinterpretq_u16_u32(c1.val[1])), + vget_high_u16(vreinterpretq_u16_u32(c3.val[1]))); +} + #endif // AV1_COMMON_ARM_TRANSPOSE_NEON_H_
diff --git a/av1/common/arm/wiener_convolve_neon.c b/av1/common/arm/wiener_convolve_neon.c new file mode 100644 index 0000000..a8f4338 --- /dev/null +++ b/av1/common/arm/wiener_convolve_neon.c
@@ -0,0 +1,375 @@ +/* + * Copyright (c) 2018, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> +#include <assert.h> + +#include "./aom_config.h" +#include "./av1_rtcd.h" +#include "aom_dsp/txfm_common.h" +#include "aom_ports/mem.h" +#include "av1/common/common.h" +#include "av1/common/arm/convolve_neon.h" +#include "av1/common/arm/mem_neon.h" +#include "av1/common/arm/transpose_neon.h" + +/* Wiener filter 2D + Apply horizontal filter and store in a temporary buffer. When applying + vertical filter, overwrite the original pixel values. + */ + +void av1_wiener_convolve_add_src_neon(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const int16_t *filter_x, int x_step_q4, + const int16_t *filter_y, int y_step_q4, + int w, int h, + const ConvolveParams *conv_params) { + uint16_t *d_tmp; + uint8_t *d; + const uint8_t *src_ptr, *s_tmp; + uint16_t *dst_ptr; + (void)x_step_q4; + (void)y_step_q4; + + int width, height; + const int bd = 8; + const int intermediate_height = h + SUBPEL_TAPS - 1; + const int center_tap = ((SUBPEL_TAPS - 1) / 2); + int16_t filter_x_tmp[7], filter_y_tmp[7]; + + DECLARE_ALIGNED(16, uint16_t, + temp[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]); + + assert(x_step_q4 == 16 && y_step_q4 == 16); + assert(w % 8); + assert(h % 4); + assert(w <= MAX_SB_SIZE); + assert(h <= MAX_SB_SIZE); + + assert(filter_x[7] == 0); + assert(filter_y[7] == 0); + + /* assumption of horizontal filtering output will not exceed 15 bit. + ((bd) + 1 + FILTER_BITS - conv_params->round_0) <= 15 + 16 - conv_params->round_0 <= 15 -- (conv_params->round_0) >= 1 + */ + assert((conv_params->round_0) >= 1); + + memcpy(&filter_x_tmp[0], filter_x, sizeof(*filter_x) * FILTER_BITS); + memcpy(&filter_y_tmp[0], filter_y, sizeof(*filter_y) * FILTER_BITS); + + filter_x_tmp[3] += (1 << FILTER_BITS); + filter_y_tmp[3] += (1 << FILTER_BITS); + + s_tmp = src - center_tap * src_stride - center_tap; + dst_ptr = temp; + src_ptr = s_tmp; + height = intermediate_height; + + /* if height is a multiple of 8 */ + if (!(h & 7)) { + int16x8_t res0, res1, res2, res3; + uint16x8_t res4, res5, res6, res7, res8, res9, res10, res11; + uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7; + uint8x8_t t8, t9, t10, t11, t12, t13, t14; + + do { + const uint8_t *s; + + __builtin_prefetch(src_ptr + 0 * src_stride); + __builtin_prefetch(src_ptr + 1 * src_stride); + __builtin_prefetch(src_ptr + 2 * src_stride); + __builtin_prefetch(src_ptr + 3 * src_stride); + __builtin_prefetch(src_ptr + 4 * src_stride); + __builtin_prefetch(src_ptr + 5 * src_stride); + __builtin_prefetch(src_ptr + 6 * src_stride); + __builtin_prefetch(src_ptr + 7 * src_stride); + + load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); + transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); + + s = src_ptr + 7; + d_tmp = dst_ptr; + width = w; + + __builtin_prefetch(dst_ptr + 0 * dst_stride); + __builtin_prefetch(dst_ptr + 1 * dst_stride); + __builtin_prefetch(dst_ptr + 2 * dst_stride); + __builtin_prefetch(dst_ptr + 3 * dst_stride); + __builtin_prefetch(dst_ptr + 4 * dst_stride); + __builtin_prefetch(dst_ptr + 5 * dst_stride); + __builtin_prefetch(dst_ptr + 6 * dst_stride); + __builtin_prefetch(dst_ptr + 7 * dst_stride); + + do { + load_u8_8x8(s, src_stride, &t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14); + transpose_u8_8x8(&t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t0, t6)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t1, t5)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t2, t4)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t3)); + res4 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t1, t7)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t2, t6)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t3, t5)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t4)); + res5 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t2, t8)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t3, t7)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t4, t6)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t5)); + res6 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t3, t9)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t4, t8)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t5, t7)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t6)); + res7 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t4, t10)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t5, t9)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t6, t8)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t7)); + res8 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t5, t11)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t6, t10)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t7, t9)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t8)); + res9 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t6, t12)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t7, t11)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t8, t10)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t9)); + res10 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + res0 = vreinterpretq_s16_u16(vaddl_u8(t7, t13)); + res1 = vreinterpretq_s16_u16(vaddl_u8(t8, t12)); + res2 = vreinterpretq_s16_u16(vaddl_u8(t9, t11)); + res3 = vreinterpretq_s16_u16(vmovl_u8(t10)); + res11 = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, + bd, conv_params->round_0); + + transpose_u16_8x8(&res4, &res5, &res6, &res7, &res8, &res9, &res10, + &res11); + store_u16_8x8(d_tmp, MAX_SB_SIZE, res4, res5, res6, res7, res8, res9, + res10, res11); + + t0 = t8; + t1 = t9; + t2 = t10; + t3 = t11; + t4 = t12; + t5 = t13; + t6 = t14; + s += 8; + d_tmp += 8; + width -= 8; + } while (width > 0); + src_ptr += 8 * src_stride; + dst_ptr += 8 * MAX_SB_SIZE; + height -= 8; + } while (height > 0); + } else { + /*if height is a multiple of 4*/ + int16x8_t tt0, tt1, tt2, tt3; + const uint8_t *s; + uint16x4_t res0, res1, res2, res3, res4, res5, res6, res7; + uint16x8_t d0, d1, d2, d3; + int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10; + int16x4_t s11, s12, s13, s14; + uint8x8_t t0, t1, t2, t3; + + do { + __builtin_prefetch(src_ptr + 0 * src_stride); + __builtin_prefetch(src_ptr + 1 * src_stride); + __builtin_prefetch(src_ptr + 2 * src_stride); + __builtin_prefetch(src_ptr + 3 * src_stride); + + load_u8_8x4(src_ptr, src_stride, &t0, &t1, &t2, &t3); /*8x4*/ + transpose_u8_8x4(&t0, &t1, &t2, + &t3); /*first 8 pixels of 4 rows transposed-- 4x8*/ + + tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); + tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); + tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); + tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); + + s0 = vget_low_s16(tt0); /*pa0 pb0 pc0 pd0 -- pixel_a0*/ + s1 = vget_low_s16(tt1); /*pa1 pb1 pc1 pd1 */ + s2 = vget_low_s16(tt2); /*pa2 pb2 pc2 pd2 */ + s3 = vget_low_s16(tt3); /*pa3 pb3 pc3 pd3 */ + s4 = vget_high_s16(tt0); /*pa4 pb4 pc4 pd4 */ + s5 = vget_high_s16(tt1); /*pa5 pb5 pc5 pd5 */ + s6 = vget_high_s16(tt2); /*pa6 pb6 pc6 pd6 */ + + __builtin_prefetch(dst_ptr + 0 * dst_stride); + __builtin_prefetch(dst_ptr + 1 * dst_stride); + __builtin_prefetch(dst_ptr + 2 * dst_stride); + __builtin_prefetch(dst_ptr + 3 * dst_stride); + + s = src_ptr + 7; + d_tmp = dst_ptr; + width = w; + + do { + load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3); /*8x4*/ + transpose_u8_8x4(&t0, &t1, &t2, &t3); + + tt0 = vreinterpretq_s16_u16(vmovl_u8(t0)); + tt1 = vreinterpretq_s16_u16(vmovl_u8(t1)); + tt2 = vreinterpretq_s16_u16(vmovl_u8(t2)); + tt3 = vreinterpretq_s16_u16(vmovl_u8(t3)); + + s7 = vget_low_s16(tt0); /*pa7 pb7 pc7 pd7 */ /*4x8*/ + s8 = vget_low_s16(tt1); /*pa8 pb8 pc8 pd8 */ + s9 = vget_low_s16(tt2); /*pa9 pb9 pc9 pd9 */ + s10 = vget_low_s16(tt3); /*pa10 pb10 pc10 pd10 */ + s11 = vget_high_s16(tt0); /*pa11 pb11 pc11 pd11 */ + s12 = vget_high_s16(tt1); /*pa12 pb12 pc12 pd12 */ + s13 = vget_high_s16(tt2); /*pa13 pb13 pc13 pd13 */ + s14 = vget_high_s16(tt3); /*pa14 pb14 pc14 pd14 */ + + res0 = wiener_convolve8_horiz_4x8( + s0, s1, s2, s3, s4, s5, s6, filter_x_tmp, bd, conv_params->round_0); + res1 = wiener_convolve8_horiz_4x8( + s1, s2, s3, s4, s5, s6, s7, filter_x_tmp, bd, conv_params->round_0); + res2 = wiener_convolve8_horiz_4x8( + s2, s3, s4, s5, s6, s7, s8, filter_x_tmp, bd, conv_params->round_0); + res3 = wiener_convolve8_horiz_4x8( + s3, s4, s5, s6, s7, s8, s9, filter_x_tmp, bd, conv_params->round_0); + res4 = + wiener_convolve8_horiz_4x8(s4, s5, s6, s7, s8, s9, s10, + filter_x_tmp, bd, conv_params->round_0); + res5 = + wiener_convolve8_horiz_4x8(s5, s6, s7, s8, s9, s10, s11, + filter_x_tmp, bd, conv_params->round_0); + res6 = + wiener_convolve8_horiz_4x8(s6, s7, s8, s9, s10, s11, s12, + filter_x_tmp, bd, conv_params->round_0); + res7 = + wiener_convolve8_horiz_4x8(s7, s8, s9, s10, s11, s12, s13, + filter_x_tmp, bd, conv_params->round_0); + + transpose_u16_4x8(&res0, &res1, &res2, &res3, &res4, &res5, &res6, + &res7, &d0, &d1, &d2, &d3); + + store_u16_8x4(d_tmp, MAX_SB_SIZE, d0, d1, d2, d3); + + s0 = s8; + s1 = s9; + s2 = s10; + s3 = s11; + s4 = s12; + s5 = s13; + s6 = s14; + s += 8; + d_tmp += 8; + width -= 8; + } while (width > 0); + + src_ptr += 4 * src_stride; + dst_ptr += 4 * MAX_SB_SIZE; + height -= 4; + } while (height > 0); + } + + { + int16x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10; + uint8x8_t t0, t1, t2, t3; + int16_t *src_tmp_ptr, *s; + uint8_t *dst_tmp_ptr; + height = h; + width = w; + src_tmp_ptr = (int16_t *)temp; + dst_tmp_ptr = dst; + src_stride = MAX_SB_SIZE; + + do { + s = src_tmp_ptr; + s0 = vld1q_s16(s); + s += src_stride; + s1 = vld1q_s16(s); + s += src_stride; + s2 = vld1q_s16(s); + s += src_stride; + s3 = vld1q_s16(s); + s += src_stride; + s4 = vld1q_s16(s); + s += src_stride; + s5 = vld1q_s16(s); + s += src_stride; + s6 = vld1q_s16(s); + s += src_stride; + d = dst_tmp_ptr; + height = h; + + do { + __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride); + __builtin_prefetch(dst_tmp_ptr + 1 * dst_stride); + __builtin_prefetch(dst_tmp_ptr + 2 * dst_stride); + __builtin_prefetch(dst_tmp_ptr + 3 * dst_stride); + + s7 = vld1q_s16(s); + s += src_stride; + s8 = vld1q_s16(s); + s += src_stride; + s9 = vld1q_s16(s); + s += src_stride; + s10 = vld1q_s16(s); + s += src_stride; + + t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp, + bd, conv_params->round_1); + t1 = wiener_convolve8_vert_4x8(s1, s2, s3, s4, s5, s6, s7, filter_y_tmp, + bd, conv_params->round_1); + t2 = wiener_convolve8_vert_4x8(s2, s3, s4, s5, s6, s7, s8, filter_y_tmp, + bd, conv_params->round_1); + t3 = wiener_convolve8_vert_4x8(s3, s4, s5, s6, s7, s8, s9, filter_y_tmp, + bd, conv_params->round_1); + + vst1_u8(d, t0); + d += dst_stride; + vst1_u8(d, t1); + d += dst_stride; + vst1_u8(d, t2); + d += dst_stride; + vst1_u8(d, t3); + d += dst_stride; + + s0 = s4; + s1 = s5; + s2 = s6; + s3 = s7; + s4 = s8; + s5 = s9; + s6 = s10; + height -= 4; + } while (height > 0); + + src_tmp_ptr += 8; + dst_tmp_ptr += 8; + + w -= 8; + } while (w > 0); + } +}
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 0c3a934..d80b671 100755 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -72,8 +72,7 @@ add_proto qw/void av1_highbd_wiener_convolve_add_src/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, const int16_t *filter_y, int y_step_q4, int w, int h, const ConvolveParams *conv_params, int bps"; -specialize qw/av1_wiener_convolve_add_src sse2/; -specialize qw/av1_wiener_convolve_add_src avx2/; +specialize qw/av1_wiener_convolve_add_src sse2 avx2 neon/; specialize qw/av1_highbd_wiener_convolve_add_src ssse3/; specialize qw/av1_highbd_wiener_convolve_add_src avx2/;
diff --git a/test/hiprec_convolve_test.cc b/test/hiprec_convolve_test.cc index 9ffb8fa..f94a073 100644 --- a/test/hiprec_convolve_test.cc +++ b/test/hiprec_convolve_test.cc
@@ -20,7 +20,6 @@ namespace { -#if HAVE_SSE2 || HAVE_AVX2 TEST_P(AV1HiprecConvolveTest, CheckOutput) { RunCheckOutput(GET_PARAM(3)); } TEST_P(AV1HiprecConvolveTest, DISABLED_SpeedTest) { RunSpeedTest(GET_PARAM(3)); @@ -35,6 +34,10 @@ libaom_test::AV1HiprecConvolve::BuildParams( av1_wiener_convolve_add_src_avx2)); #endif +#if HAVE_NEON +INSTANTIATE_TEST_CASE_P(NEON, AV1HiprecConvolveTest, + libaom_test::AV1HiprecConvolve::BuildParams( + av1_wiener_convolve_add_src_neon)); #endif #if HAVE_SSSE3 || HAVE_AVX2
diff --git a/test/hiprec_convolve_test_util.cc b/test/hiprec_convolve_test_util.cc index 860ed4c..673e61b 100644 --- a/test/hiprec_convolve_test_util.cc +++ b/test/hiprec_convolve_test_util.cc
@@ -52,9 +52,10 @@ ::testing::internal::ParamGenerator<HiprecConvolveParam> BuildParams( hiprec_convolve_func filter) { const HiprecConvolveParam params[] = { - make_tuple(8, 8, 50000, filter), - make_tuple(64, 64, 1000, filter), - make_tuple(32, 8, 10000, filter), + make_tuple(8, 8, 50000, filter), make_tuple(8, 4, 50000, filter), + make_tuple(64, 24, 1000, filter), make_tuple(64, 64, 1000, filter), + make_tuple(64, 56, 1000, filter), make_tuple(32, 8, 10000, filter), + make_tuple(32, 28, 10000, filter), make_tuple(32, 32, 10000, filter) }; return ::testing::ValuesIn(params); }
diff --git a/test/test.cmake b/test/test.cmake index faf2fd0..0bb3969 100644 --- a/test/test.cmake +++ b/test/test.cmake
@@ -108,7 +108,10 @@ "${AOM_ROOT}/test/cdef_test.cc" "${AOM_ROOT}/test/intrabc_test.cc" "${AOM_ROOT}/test/intrapred_test.cc" "${AOM_ROOT}/test/lpf_test.cc" "${AOM_ROOT}/test/onyxc_int_test.cc" "${AOM_ROOT}/test/scan_test.cc" - "${AOM_ROOT}/test/simd_cmp_impl.h") + "${AOM_ROOT}/test/simd_cmp_impl.h" + "${AOM_ROOT}/test/hiprec_convolve_test.cc" + "${AOM_ROOT}/test/hiprec_convolve_test_util.cc" + "${AOM_ROOT}/test/hiprec_convolve_test_util.h") set(AOM_UNIT_TEST_ENCODER_SOURCES ${AOM_UNIT_TEST_ENCODER_SOURCES} "${AOM_ROOT}/test/motion_vector_test.cc") @@ -124,13 +127,6 @@ set(AOM_UNIT_TEST_COMMON_SOURCES ${AOM_UNIT_TEST_COMMON_SOURCES} "${AOM_ROOT}/test/selfguided_filter_test.cc") - if(HAVE_SSE2) - set(AOM_UNIT_TEST_COMMON_SOURCES ${AOM_UNIT_TEST_COMMON_SOURCES} - "${AOM_ROOT}/test/hiprec_convolve_test.cc" - "${AOM_ROOT}/test/hiprec_convolve_test_util.cc" - "${AOM_ROOT}/test/hiprec_convolve_test_util.h") - endif() - set(AOM_UNIT_TEST_ENCODER_SOURCES ${AOM_UNIT_TEST_ENCODER_SOURCES} "${AOM_ROOT}/test/encodetxb_test.cc")