Refactor Neon compound convolution functions 5/7 Reafactor the helper functions implementing the vertical pass of av1_dist_wtd_convolve_2d_neon for both 6-tap and 8-tap filters. This is mostly just cosmetic changes, as well as moving the convolution kernel helper functions into the .c file from the convolve_neon.h header. Change-Id: I8b163ba4478d163608061278f491120a5f39e3e7
diff --git a/av1/common/arm/convolve_neon.h b/av1/common/arm/convolve_neon.h index 520a393..cbca376 100644 --- a/av1/common/arm/convolve_neon.h +++ b/av1/common/arm/convolve_neon.h
@@ -508,108 +508,6 @@ return sum; } -static INLINE uint16x4_t convolve6_4_s32(const int16x4_t s0, const int16x4_t s1, - const int16x4_t s2, const int16x4_t s3, - const int16x4_t s4, const int16x4_t s5, - const int16x8_t y_filter, - const int32x4_t offset_const) { - const int16x4_t y_filter_lo = vget_low_s16(y_filter); - const int16x4_t y_filter_hi = vget_high_s16(y_filter); - - int32x4_t sum = offset_const; - sum = vmlal_lane_s16(sum, s0, y_filter_lo, 1); - sum = vmlal_lane_s16(sum, s1, y_filter_lo, 2); - sum = vmlal_lane_s16(sum, s2, y_filter_lo, 3); - sum = vmlal_lane_s16(sum, s3, y_filter_hi, 0); - sum = vmlal_lane_s16(sum, s4, y_filter_hi, 1); - sum = vmlal_lane_s16(sum, s5, y_filter_hi, 2); - - return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS); -} - -static INLINE uint16x8_t convolve6_8_s32(const int16x8_t s0, const int16x8_t s1, - const int16x8_t s2, const int16x8_t s3, - const int16x8_t s4, const int16x8_t s5, - const int16x8_t y_filter, - const int32x4_t offset_const) { - const int16x4_t y_filter_lo = vget_low_s16(y_filter); - const int16x4_t y_filter_hi = vget_high_s16(y_filter); - - int32x4_t sum0 = offset_const; - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_lo, 1); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 2); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 3); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_hi, 0); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 1); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 2); - - int32x4_t sum1 = offset_const; - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_lo, 1); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 2); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 3); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_hi, 0); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 1); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 2); - - return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS), - vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS)); -} - -static INLINE uint16x4_t convolve8_4_s32(const int16x4_t s0, const int16x4_t s1, - const int16x4_t s2, const int16x4_t s3, - const int16x4_t s4, const int16x4_t s5, - const int16x4_t s6, const int16x4_t s7, - const int16x8_t y_filter, - const int32x4_t offset_const) { - const int16x4_t y_filter_lo = vget_low_s16(y_filter); - const int16x4_t y_filter_hi = vget_high_s16(y_filter); - - int32x4_t sum = offset_const; - sum = vmlal_lane_s16(sum, s0, y_filter_lo, 0); - sum = vmlal_lane_s16(sum, s1, y_filter_lo, 1); - sum = vmlal_lane_s16(sum, s2, y_filter_lo, 2); - sum = vmlal_lane_s16(sum, s3, y_filter_lo, 3); - sum = vmlal_lane_s16(sum, s4, y_filter_hi, 0); - sum = vmlal_lane_s16(sum, s5, y_filter_hi, 1); - sum = vmlal_lane_s16(sum, s6, y_filter_hi, 2); - sum = vmlal_lane_s16(sum, s7, y_filter_hi, 3); - - return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS); -} - -static INLINE uint16x8_t convolve8_8_s32(const int16x8_t s0, const int16x8_t s1, - const int16x8_t s2, const int16x8_t s3, - const int16x8_t s4, const int16x8_t s5, - const int16x8_t s6, const int16x8_t s7, - const int16x8_t y_filter, - const int32x4_t offset_const) { - const int16x4_t y_filter_lo = vget_low_s16(y_filter); - const int16x4_t y_filter_hi = vget_high_s16(y_filter); - - int32x4_t sum0 = offset_const; - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_lo, 0); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_lo, 1); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_lo, 2); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_lo, 3); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_hi, 0); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_hi, 1); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_hi, 2); - sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_hi, 3); - - int32x4_t sum1 = offset_const; - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_lo, 0); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_lo, 1); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_lo, 2); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_lo, 3); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_hi, 0); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_hi, 1); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_hi, 2); - sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_hi, 3); - - return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS), - vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS)); -} - #if !(defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD)) static INLINE int16x4_t convolve8_horiz_4x4_s16(
diff --git a/av1/common/arm/jnt_convolve_neon.c b/av1/common/arm/jnt_convolve_neon.c index af4cf8f..02f6370 100644 --- a/av1/common/arm/jnt_convolve_neon.c +++ b/av1/common/arm/jnt_convolve_neon.c
@@ -592,28 +592,75 @@ #endif // defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) +static INLINE uint16x4_t +convolve6_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, + const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, + const int16x8_t y_filter, const int32x4_t offset_const) { + const int16x4_t y_filter_0_3 = vget_low_s16(y_filter); + const int16x4_t y_filter_4_7 = vget_high_s16(y_filter); + + int32x4_t sum = offset_const; + // Filter values at indices 0 and 7 are 0. + sum = vmlal_lane_s16(sum, s0, y_filter_0_3, 1); + sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 2); + sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 3); + sum = vmlal_lane_s16(sum, s3, y_filter_4_7, 0); + sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 1); + sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 2); + + return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS); +} + +static INLINE uint16x8_t +convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, + const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, + const int16x8_t y_filter, const int32x4_t offset_const) { + const int16x4_t y_filter_0_3 = vget_low_s16(y_filter); + const int16x4_t y_filter_4_7 = vget_high_s16(y_filter); + + int32x4_t sum0 = offset_const; + // Filter values at indices 0 and 7 are 0. + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_0_3, 1); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 2); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 3); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_4_7, 0); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 1); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 2); + + int32x4_t sum1 = offset_const; + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_0_3, 1); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 2); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 3); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_4_7, 0); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 1); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 2); + + return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS), + vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS)); +} + static INLINE void dist_wtd_convolve_2d_vert_6tap_neon( int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride, ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) { - CONV_BUF_TYPE *dst_ptr = conv_params->dst; - const int dst_stride = conv_params->dst_stride; - const int bd = 8; const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits); const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); - const uint16_t fwd_offset = conv_params->fwd_offset; - const uint16_t bck_offset = conv_params->bck_offset; + const int do_average = conv_params->do_average; const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; if (w == 4) { int16x4_t s0, s1, s2, s3, s4, s5; uint16x4_t dd0, d0; uint8x8_t d01_u8; - #if defined(__aarch64__) int16x4_t s6, s7, s8; uint16x4_t dd1, dd2, dd3, d1, d2, d3; @@ -627,10 +674,10 @@ #if defined(__aarch64__) load_s16_4x4(src_ptr, src_stride, &s5, &s6, &s7, &s8); - d0 = convolve6_4_s32(s0, s1, s2, s3, s4, s5, y_filter, offset_const); - d1 = convolve6_4_s32(s1, s2, s3, s4, s5, s6, y_filter, offset_const); - d2 = convolve6_4_s32(s2, s3, s4, s5, s6, s7, y_filter, offset_const); - d3 = convolve6_4_s32(s3, s4, s5, s6, s7, s8, y_filter, offset_const); + d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d1 = convolve6_4_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const); + d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const); + d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const); if (do_average) { load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); @@ -659,7 +706,7 @@ #else // !defined(__aarch64__) s5 = vld1_s16(src_ptr); - d0 = convolve6_4_s32(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const); if (do_average) { dd0 = vld1_u16(dst_ptr); @@ -683,13 +730,11 @@ dst_ptr += dst_stride; h--; #endif // defined(__aarch64__) - } while (h > 0); - + } while (h != 0); } else { int16x8_t s0, s1, s2, s3, s4, s5; uint16x8_t dd0, d0; uint8x8_t d0_u8; - #if defined(__aarch64__) int16x8_t s6, s7, s8; uint16x8_t dd1, dd2, dd3, d1, d2, d3; @@ -698,7 +743,7 @@ do { int16_t *s = src_ptr; - uint16_t *d = dst_ptr; + CONV_BUF_TYPE *d = dst_ptr; uint8_t *d_u8 = dst8_ptr; int height = h; @@ -709,10 +754,10 @@ #if defined(__aarch64__) load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8); - d0 = convolve6_8_s32(s0, s1, s2, s3, s4, s5, y_filter, offset_const); - d1 = convolve6_8_s32(s1, s2, s3, s4, s5, s6, y_filter, offset_const); - d2 = convolve6_8_s32(s2, s3, s4, s5, s6, s7, y_filter, offset_const); - d3 = convolve6_8_s32(s3, s4, s5, s6, s7, s8, y_filter, offset_const); + d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d1 = convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, offset_const); + d2 = convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, offset_const); + d3 = convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, offset_const); if (do_average) { load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); @@ -738,7 +783,7 @@ #else // !defined(__aarch64__) s5 = vld1q_s16(s); - d0 = convolve6_8_s32(s0, s1, s2, s3, s4, s5, y_filter, offset_const); + d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, offset_const); if (do_average) { dd0 = vld1q_u16(d); @@ -757,43 +802,94 @@ s2 = s3; s3 = s4; s4 = s5; - s += src_stride; d += dst_stride; height--; #endif // defined(__aarch64__) - } while (height > 0); - + } while (height != 0); src_ptr += 8; dst_ptr += 8; dst8_ptr += 8; w -= 8; - } while (w > 0); + } while (w != 0); } } +static INLINE uint16x4_t +convolve8_4_2d_v(const int16x4_t s0, const int16x4_t s1, const int16x4_t s2, + const int16x4_t s3, const int16x4_t s4, const int16x4_t s5, + const int16x4_t s6, const int16x4_t s7, + const int16x8_t y_filter, const int32x4_t offset_const) { + const int16x4_t y_filter_0_3 = vget_low_s16(y_filter); + const int16x4_t y_filter_4_7 = vget_high_s16(y_filter); + + int32x4_t sum = offset_const; + sum = vmlal_lane_s16(sum, s0, y_filter_0_3, 0); + sum = vmlal_lane_s16(sum, s1, y_filter_0_3, 1); + sum = vmlal_lane_s16(sum, s2, y_filter_0_3, 2); + sum = vmlal_lane_s16(sum, s3, y_filter_0_3, 3); + sum = vmlal_lane_s16(sum, s4, y_filter_4_7, 0); + sum = vmlal_lane_s16(sum, s5, y_filter_4_7, 1); + sum = vmlal_lane_s16(sum, s6, y_filter_4_7, 2); + sum = vmlal_lane_s16(sum, s7, y_filter_4_7, 3); + + return vqrshrun_n_s32(sum, COMPOUND_ROUND1_BITS); +} + +static INLINE uint16x8_t +convolve8_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, + const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, + const int16x8_t s6, const int16x8_t s7, + const int16x8_t y_filter, const int32x4_t offset_const) { + const int16x4_t y_filter_0_3 = vget_low_s16(y_filter); + const int16x4_t y_filter_4_7 = vget_high_s16(y_filter); + + int32x4_t sum0 = offset_const; + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s0), y_filter_0_3, 0); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter_0_3, 1); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter_0_3, 2); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s3), y_filter_0_3, 3); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s4), y_filter_4_7, 0); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s5), y_filter_4_7, 1); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s6), y_filter_4_7, 2); + sum0 = vmlal_lane_s16(sum0, vget_low_s16(s7), y_filter_4_7, 3); + + int32x4_t sum1 = offset_const; + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s0), y_filter_0_3, 0); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s1), y_filter_0_3, 1); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s2), y_filter_0_3, 2); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s3), y_filter_0_3, 3); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s4), y_filter_4_7, 0); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s5), y_filter_4_7, 1); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s6), y_filter_4_7, 2); + sum1 = vmlal_lane_s16(sum1, vget_high_s16(s7), y_filter_4_7, 3); + + return vcombine_u16(vqrshrun_n_s32(sum0, COMPOUND_ROUND1_BITS), + vqrshrun_n_s32(sum1, COMPOUND_ROUND1_BITS)); +} + static INLINE void dist_wtd_convolve_2d_vert_8tap_neon( int16_t *src_ptr, const int src_stride, uint8_t *dst8_ptr, int dst8_stride, ConvolveParams *conv_params, const int16x8_t y_filter, int h, int w) { - CONV_BUF_TYPE *dst_ptr = conv_params->dst; - const int dst_stride = conv_params->dst_stride; - const int bd = 8; const int offset_bits = bd + 2 * FILTER_BITS - ROUND0_BITS; const int32x4_t offset_const = vdupq_n_s32(1 << offset_bits); const int16_t round_offset = (1 << (offset_bits - COMPOUND_ROUND1_BITS)) + (1 << (offset_bits - COMPOUND_ROUND1_BITS - 1)); const int16x8_t round_offset_vec = vdupq_n_s16(round_offset); - const uint16_t fwd_offset = conv_params->fwd_offset; - const uint16_t bck_offset = conv_params->bck_offset; + const int do_average = conv_params->do_average; const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; + const uint16_t fwd_offset = conv_params->fwd_offset; + const uint16_t bck_offset = conv_params->bck_offset; + + CONV_BUF_TYPE *dst_ptr = conv_params->dst; + const int dst_stride = conv_params->dst_stride; if (w == 4) { int16x4_t s0, s1, s2, s3, s4, s5, s6, s7; uint16x4_t dd0, d0; uint8x8_t d01_u8; - #if defined(__aarch64__) int16x4_t s8, s9, s10; uint16x4_t dd1, dd2, dd3, d1, d2, d3; @@ -807,14 +903,14 @@ #if defined(__aarch64__) load_s16_4x4(src_ptr, src_stride, &s7, &s8, &s9, &s10); - d0 = convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - offset_const); - d1 = convolve8_4_s32(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, - offset_const); - d2 = convolve8_4_s32(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, - offset_const); - d3 = convolve8_4_s32(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, - offset_const); + d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); + d1 = convolve8_4_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, + offset_const); + d2 = convolve8_4_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, + offset_const); + d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, + offset_const); if (do_average) { load_u16_4x4(dst_ptr, dst_stride, &dd0, &dd1, &dd2, &dd3); @@ -845,8 +941,8 @@ #else // !defined(__aarch64__) s7 = vld1_s16(src_ptr); - d0 = convolve8_4_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - offset_const); + d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); if (do_average) { dd0 = vld1_u16(dst_ptr); @@ -872,13 +968,11 @@ dst_ptr += dst_stride; h--; #endif // defined(__aarch64__) - } while (h > 0); - + } while (h != 0); } else { int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; uint16x8_t dd0, d0; uint8x8_t d0_u8; - #if defined(__aarch64__) int16x8_t s8, s9, s10; uint16x8_t dd1, dd2, dd3, d1, d2, d3; @@ -887,7 +981,7 @@ do { int16_t *s = src_ptr; - uint16_t *d = dst_ptr; + CONV_BUF_TYPE *d = dst_ptr; uint8_t *d_u8 = dst8_ptr; int height = h; @@ -898,14 +992,14 @@ #if defined(__aarch64__) load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10); - d0 = convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - offset_const); - d1 = convolve8_8_s32(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, - offset_const); - d2 = convolve8_8_s32(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, - offset_const); - d3 = convolve8_8_s32(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, - offset_const); + d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); + d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter, + offset_const); + d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter, + offset_const); + d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter, + offset_const); if (do_average) { load_u16_8x4(d, dst_stride, &dd0, &dd1, &dd2, &dd3); @@ -933,8 +1027,8 @@ #else // !defined(__aarch64__) s7 = vld1q_s16(s); - d0 = convolve8_8_s32(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, - offset_const); + d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter, + offset_const); if (do_average) { dd0 = vld1q_u16(d); @@ -959,13 +1053,12 @@ d += dst_stride; height--; #endif // defined(__aarch64__) - } while (height > 0); - + } while (height != 0); src_ptr += 8; dst_ptr += 8; dst8_ptr += 8; w -= 8; - } while (w > 0); + } while (w != 0); } }