Simplify av1_convolve_2d_sr_* Arm SIMD kernels Remove some redundant constants added in the horizontal pass and subtracted in the vertical pass of the Arm SIMD implementations of av1_convolve_2d_sr_*. This patch is mostly a port of the following SVT-AV1 commit: https://gitlab.com/AOMediaCodec/SVT-AV1/-/commit/0c7d5a9f0e099c685d0a7a071bb026864782b841 Change-Id: Ib7eb9329f25a700784bdb02fa602a8a38dbc7d30
diff --git a/av1/common/arm/convolve_neon.c b/av1/common/arm/convolve_neon.c index bc7c711..fc42f9e 100644 --- a/av1/common/arm/convolve_neon.c +++ b/av1/common/arm/convolve_neon.c
@@ -1188,11 +1188,9 @@ const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr, const int dst_stride, int w, int h, const int16x8_t x_filter_0_7, const int16x4_t x_filter_8_11) { - const int bd = 8; // A shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts - // which are generally faster than rounding shifts on modern CPUs. - const int32x4_t horiz_const = - vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1)); #if AOM_ARCH_AARCH64 do { @@ -1322,7 +1320,6 @@ static inline void convolve_2d_sr_horiz_4tap_neon( const uint8_t *src, ptrdiff_t src_stride, int16_t *dst, ptrdiff_t dst_stride, int w, int h, const int16_t *filter_x) { - const int bd = 8; // All filter values are even, halve to reduce intermediate precision // requirements. const int16x4_t filter = vshr_n_s16(vld1_s16(filter_x + 2), 1); @@ -1330,8 +1327,7 @@ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // (The extra -1 is needed because we halved the filter values.) - const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) + - (1 << ((ROUND0_BITS - 1) - 1))); + const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1)); if (w == 4) { do { @@ -1451,8 +1447,6 @@ static inline void convolve_2d_sr_horiz_8tap_neon( const uint8_t *src, int src_stride, int16_t *im_block, int im_stride, int w, int im_h, const int16_t *x_filter_ptr) { - const int bd = 8; - const uint8_t *src_ptr = src; int16_t *dst_ptr = im_block; int dst_stride = im_stride; @@ -1461,8 +1455,7 @@ // A shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // (The extra -1 is needed because we halved the filter values.) - const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) + - (1 << ((ROUND0_BITS - 1) - 1))); + const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1)); // Filter values are even, so halve to reduce intermediate precision reqs. const int16x8_t x_filter = vshrq_n_s16(vld1q_s16(x_filter_ptr), 1);
diff --git a/av1/common/arm/convolve_neon.h b/av1/common/arm/convolve_neon.h index 206f3ba..9d6e2fb 100644 --- a/av1/common/arm/convolve_neon.h +++ b/av1/common/arm/convolve_neon.h
@@ -50,8 +50,7 @@ const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, const int16x8_t s6, const int16x8_t s7, const int16x8_t s8, const int16x8_t s9, const int16x8_t s10, const int16x8_t s11, - const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11, - const int16x8_t sub_const) { + const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) { const int16x4_t y_filter_0_3 = vget_low_s16(y_filter_0_7); const int16x4_t y_filter_4_7 = vget_high_s16(y_filter_0_7); @@ -84,7 +83,6 @@ int16x8_t res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS), vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS)); - res = vsubq_s16(res, sub_const); return vqmovun_s16(res); } @@ -92,9 +90,6 @@ static inline void convolve_2d_sr_vert_12tap_neon( int16_t *src_ptr, int src_stride, uint8_t *dst_ptr, int dst_stride, int w, int h, const int16x8_t y_filter_0_7, const int16x4_t y_filter_8_11) { - const int bd = 8; - const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - if (w <= 4) { int16x4_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10; load_s16_4x11(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, @@ -122,9 +117,6 @@ vcombine_s16(vqrshrn_n_s32(d2, 2 * FILTER_BITS - ROUND0_BITS), vqrshrn_n_s32(d3, 2 * FILTER_BITS - ROUND0_BITS)); - dd01 = vsubq_s16(dd01, sub_const); - dd23 = vsubq_s16(dd23, sub_const); - uint8x8_t d01 = vqmovun_s16(dd01); uint8x8_t d23 = vqmovun_s16(dd23); @@ -162,18 +154,17 @@ int16x8_t s11, s12, s13, s14; load_s16_8x4(s, src_stride, &s11, &s12, &s13, &s14); - uint8x8_t d0 = - convolve12_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, - y_filter_0_7, y_filter_8_11, sub_const); + uint8x8_t d0 = convolve12_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, + s10, s11, y_filter_0_7, y_filter_8_11); uint8x8_t d1 = convolve12_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, - y_filter_0_7, y_filter_8_11, sub_const); + y_filter_0_7, y_filter_8_11); uint8x8_t d2 = convolve12_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, - s13, y_filter_0_7, y_filter_8_11, sub_const); + s13, y_filter_0_7, y_filter_8_11); uint8x8_t d3 = convolve12_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, - s14, y_filter_0_7, y_filter_8_11, sub_const); + s14, y_filter_0_7, y_filter_8_11); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -223,8 +214,7 @@ const int16x8_t s2, const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, const int16x8_t s6, const int16x8_t s7, - const int16x8_t y_filter, - const int16x8_t sub_const) { + const int16x8_t y_filter) { const int16x4_t y_filter_lo = vget_low_s16(y_filter); const int16x4_t y_filter_hi = vget_high_s16(y_filter); @@ -249,7 +239,6 @@ int16x8_t res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS), vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS)); - res = vsubq_s16(res, sub_const); return vqmovun_s16(res); } @@ -259,9 +248,6 @@ uint8_t *dst_ptr, int dst_stride, int w, int h, const int16x8_t y_filter) { - const int bd = 8; - const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - if (w <= 4) { int16x4_t s0, s1, s2, s3, s4, s5, s6; load_s16_4x7(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4, &s5, &s6); @@ -278,8 +264,8 @@ int16x4_t d3 = convolve8_4_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter); - uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const)); - uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); + uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -297,8 +283,7 @@ #else // !AOM_ARCH_AARCH64 int16x4_t s7 = vld1_s16(src_ptr); int16x4_t d0 = convolve8_4_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); - uint8x8_t d01 = - vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, vdup_n_s16(0))); store_u8_4x1(dst_ptr, d01); @@ -330,14 +315,14 @@ int16x8_t s7, s8, s9, s10; load_s16_8x4(s, src_stride, &s7, &s8, &s9, &s10); - uint8x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, - y_filter, sub_const); - uint8x8_t d1 = convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, - y_filter, sub_const); - uint8x8_t d2 = convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, - y_filter, sub_const); - uint8x8_t d3 = convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, - y_filter, sub_const); + uint8x8_t d0 = + convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); + uint8x8_t d1 = + convolve8_8_2d_v(s1, s2, s3, s4, s5, s6, s7, s8, y_filter); + uint8x8_t d2 = + convolve8_8_2d_v(s2, s3, s4, s5, s6, s7, s8, s9, y_filter); + uint8x8_t d3 = + convolve8_8_2d_v(s3, s4, s5, s6, s7, s8, s9, s10, y_filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -353,8 +338,8 @@ height -= 4; #else // !AOM_ARCH_AARCH64 int16x8_t s7 = vld1q_s16(s); - uint8x8_t d0 = convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, - y_filter, sub_const); + uint8x8_t d0 = + convolve8_8_2d_v(s0, s1, s2, s3, s4, s5, s6, s7, y_filter); vst1_u8(d, d0); s0 = s1; @@ -396,8 +381,7 @@ static inline uint8x8_t convolve6_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, const int16x8_t s3, const int16x8_t s4, const int16x8_t s5, - const int16x8_t y_filter, - const int16x8_t sub_const) { + const int16x8_t y_filter) { const int16x4_t y_filter_lo = vget_low_s16(y_filter); const int16x4_t y_filter_hi = vget_high_s16(y_filter); @@ -418,7 +402,6 @@ int16x8_t res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS), vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS)); - res = vsubq_s16(res, sub_const); return vqmovun_s16(res); } @@ -428,9 +411,6 @@ uint8_t *dst_ptr, int dst_stride, int w, int h, const int16x8_t y_filter) { - const int bd = 8; - const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - if (w <= 4) { int16x4_t s0, s1, s2, s3, s4; load_s16_4x5(src_ptr, src_stride, &s0, &s1, &s2, &s3, &s4); @@ -446,8 +426,8 @@ int16x4_t d2 = convolve6_4_2d_v(s2, s3, s4, s5, s6, s7, y_filter); int16x4_t d3 = convolve6_4_2d_v(s3, s4, s5, s6, s7, s8, y_filter); - uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const)); - uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); + uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -463,8 +443,7 @@ #else // !AOM_ARCH_AARCH64 int16x4_t s5 = vld1_s16(src_ptr); int16x4_t d0 = convolve6_4_2d_v(s0, s1, s2, s3, s4, s5, y_filter); - uint8x8_t d01 = - vqmovun_s16(vsubq_s16(vcombine_s16(d0, vdup_n_s16(0)), sub_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, vdup_n_s16(0))); store_u8_4x1(dst_ptr, d01); @@ -494,14 +473,10 @@ int16x8_t s5, s6, s7, s8; load_s16_8x4(s, src_stride, &s5, &s6, &s7, &s8); - uint8x8_t d0 = - convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, sub_const); - uint8x8_t d1 = - convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter, sub_const); - uint8x8_t d2 = - convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter, sub_const); - uint8x8_t d3 = - convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter, sub_const); + uint8x8_t d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter); + uint8x8_t d1 = convolve6_8_2d_v(s1, s2, s3, s4, s5, s6, y_filter); + uint8x8_t d2 = convolve6_8_2d_v(s2, s3, s4, s5, s6, s7, y_filter); + uint8x8_t d3 = convolve6_8_2d_v(s3, s4, s5, s6, s7, s8, y_filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -515,8 +490,7 @@ height -= 4; #else // !AOM_ARCH_AARCH64 int16x8_t s5 = vld1q_s16(s); - uint8x8_t d0 = - convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter, sub_const); + uint8x8_t d0 = convolve6_8_2d_v(s0, s1, s2, s3, s4, s5, y_filter); vst1_u8(d, d0); s0 = s1; @@ -549,8 +523,7 @@ static inline uint8x8_t convolve4_8_2d_v(const int16x8_t s0, const int16x8_t s1, const int16x8_t s2, const int16x8_t s3, - const int16x4_t y_filter, - const int16x8_t sub_const) { + const int16x4_t y_filter) { int32x4_t sum0 = vmull_lane_s16(vget_low_s16(s0), y_filter, 0); sum0 = vmlal_lane_s16(sum0, vget_low_s16(s1), y_filter, 1); sum0 = vmlal_lane_s16(sum0, vget_low_s16(s2), y_filter, 2); @@ -564,7 +537,6 @@ int16x8_t res = vcombine_s16(vqrshrn_n_s32(sum0, 2 * FILTER_BITS - ROUND0_BITS), vqrshrn_n_s32(sum1, 2 * FILTER_BITS - ROUND0_BITS)); - res = vsubq_s16(res, sub_const); return vqmovun_s16(res); } @@ -574,9 +546,6 @@ uint8_t *dst_ptr, int dst_stride, int w, int h, const int16_t *y_filter) { - const int bd = 8; - const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - const int16x4_t filter = vld1_s16(y_filter + 2); if (w == 4) { @@ -593,8 +562,8 @@ int16x4_t d2 = convolve4_4_2d_v(s2, s3, s4, s5, filter); int16x4_t d3 = convolve4_4_2d_v(s3, s4, s5, s6, filter); - uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), sub_const)); - uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), sub_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); + uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); store_u8x4_strided_x2(dst_ptr + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst_ptr + 2 * dst_stride, dst_stride, d23); @@ -622,10 +591,10 @@ int16x8_t s3, s4, s5, s6; load_s16_8x4(s, src_stride, &s3, &s4, &s5, &s6); - uint8x8_t d0 = convolve4_8_2d_v(s0, s1, s2, s3, filter, sub_const); - uint8x8_t d1 = convolve4_8_2d_v(s1, s2, s3, s4, filter, sub_const); - uint8x8_t d2 = convolve4_8_2d_v(s2, s3, s4, s5, filter, sub_const); - uint8x8_t d3 = convolve4_8_2d_v(s3, s4, s5, s6, filter, sub_const); + uint8x8_t d0 = convolve4_8_2d_v(s0, s1, s2, s3, filter); + uint8x8_t d1 = convolve4_8_2d_v(s1, s2, s3, s4, filter); + uint8x8_t d2 = convolve4_8_2d_v(s2, s3, s4, s5, filter); + uint8x8_t d3 = convolve4_8_2d_v(s3, s4, s5, s6, filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
diff --git a/av1/common/arm/convolve_neon_dotprod.c b/av1/common/arm/convolve_neon_dotprod.c index 35cdf9e..b105e01 100644 --- a/av1/common/arm/convolve_neon_dotprod.c +++ b/av1/common/arm/convolve_neon_dotprod.c
@@ -1065,8 +1065,6 @@ // The no-op filter should never be used here. assert(vgetq_lane_s16(x_filter_0_7, 5) != 128); - const int bd = 8; - // Narrow filter values to 8-bit. const int16x8x2_t x_filter_s16 = { { x_filter_0_7, vcombine_s16(x_filter_8_11, vdup_n_s16(0)) } @@ -1076,8 +1074,7 @@ // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. - const int32_t horiz_const = - ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32_t horiz_const = 1 << (ROUND0_BITS - 1); // Dot product constants. const int32x4_t correction = vdupq_n_s32((128 << FILTER_BITS) + horiz_const); const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); @@ -1203,7 +1200,6 @@ static inline void convolve_2d_sr_horiz_4tap_neon_dotprod( const uint8_t *src, ptrdiff_t src_stride, int16_t *dst, ptrdiff_t dst_stride, int w, int h, const int16_t *filter_x) { - const int bd = 8; const int16x4_t x_filter = vld1_s16(filter_x + 2); // All 4-tap and bilinear filter values are even, so halve them to reduce // intermediate precision requirements. @@ -1211,8 +1207,7 @@ // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. - const int32_t horiz_const = - ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32_t horiz_const = 1 << (ROUND0_BITS - 1); // Accumulate into 128 << FILTER_BITS to account for range transform. // Halve the total because we halved the filter values. const int32x4_t correction = @@ -1328,11 +1323,9 @@ // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = vshrn_n_s16(x_filter_s16, 1); - const int bd = 8; // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. - const int32_t horiz_const = - ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32_t horiz_const = 1 << (ROUND0_BITS - 1); // Halve the total because we halved the filter values. const int32x4_t correction = vdupq_n_s32(((128 << FILTER_BITS) + horiz_const) / 2); @@ -1394,16 +1387,13 @@ // Filter values are even, so halve to reduce intermediate precision reqs. const int8x8_t x_filter = vshrn_n_s16(vld1q_s16(x_filter_ptr), 1); - const int bd = 8; // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. - const int32_t horiz_const = - ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32_t horiz_const = 1 << (ROUND0_BITS - 1); // Accumulate into 128 << FILTER_BITS to account for range transform. // Halve the total because we halved the filter values. const int32x4_t correction = vdupq_n_s32(((128 << FILTER_BITS) + horiz_const) / 2); - const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1)); const uint8x16x3_t permute_tbl = vld1q_u8_x3(kDotProdPermuteTbl); do { @@ -1434,14 +1424,14 @@ int16x8_t v_s8 = convolve8_8_2d_h(h_s8, x_filter, correction, permute_tbl); - uint8x8_t d0 = convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, - y_filter, vert_const); - uint8x8_t d1 = convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, - y_filter, vert_const); - uint8x8_t d2 = convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7, - y_filter, vert_const); - uint8x8_t d3 = convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8, - y_filter, vert_const); + uint8x8_t d0 = + convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, y_filter); + uint8x8_t d1 = + convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, y_filter); + uint8x8_t d2 = + convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7, y_filter); + uint8x8_t d3 = + convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8, y_filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -1464,9 +1454,6 @@ static inline void convolve_2d_sr_4tap_neon_dotprod( const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const int16_t *x_filter_ptr, const int16_t *y_filter_ptr) { - const int bd = 8; - const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1)); - const int16x4_t y_filter = vld1_s16(y_filter_ptr + 2); const int16x4_t x_filter_s16 = vld1_s16(x_filter_ptr + 2); // All 4-tap and bilinear filter values are even, so halve them to reduce @@ -1476,8 +1463,7 @@ // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. - const int32_t horiz_const = - ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32_t horiz_const = 1 << (ROUND0_BITS - 1); // Accumulate into 128 << FILTER_BITS to account for range transform. // Halve the total because we halved the filter values. const int32x4_t correction = @@ -1513,8 +1499,8 @@ int16x4_t d2 = convolve4_4_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter); int16x4_t d3 = convolve4_4_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter); - uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), vert_const)); - uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), vert_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); + uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -1560,14 +1546,10 @@ int16x8_t v_s6 = convolve4_8_2d_h(h_s6, x_filter, permute_tbl, correction); - uint8x8_t d0 = - convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter, vert_const); - uint8x8_t d1 = - convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter, vert_const); - uint8x8_t d2 = - convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter, vert_const); - uint8x8_t d3 = - convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter, vert_const); + uint8x8_t d0 = convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter); + uint8x8_t d1 = convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter); + uint8x8_t d2 = convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter); + uint8x8_t d3 = convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
diff --git a/av1/common/arm/convolve_neon_i8mm.c b/av1/common/arm/convolve_neon_i8mm.c index 1452841..0e04060 100644 --- a/av1/common/arm/convolve_neon_i8mm.c +++ b/av1/common/arm/convolve_neon_i8mm.c
@@ -931,12 +931,10 @@ const uint8x8_t f0 = vdup_n_u8(-x_filter_ptr[0] >> 1); const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul8PermuteTbl); - const int bd = 8; // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // The outermost -1 is needed because we halved the filter values. - const int16x8_t horiz_const = vdupq_n_s16((1 << (bd + FILTER_BITS - 2)) + - (1 << ((ROUND0_BITS - 1) - 1))); + const int16x8_t horiz_const = vdupq_n_s16(1 << ((ROUND0_BITS - 1) - 1)); const uint8_t *src_ptr = src; int16_t *dst_ptr = im_block; @@ -1030,7 +1028,6 @@ static inline void convolve_2d_sr_horiz_4tap_neon_i8mm( const uint8_t *src, int src_stride, int16_t *dst, int dst_stride, int width, int height, const int16_t *filter_x) { - const int bd = 8; const int16x4_t x_filter = vld1_s16(filter_x + 2); // All 4-tap and bilinear filter values are even, so halve them to reduce // intermediate precision requirements. @@ -1039,8 +1036,7 @@ // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // Halve the total because we halved the filter values. - const int32x4_t horiz_const = vdupq_n_s32( - (((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))) / 2)); + const int32x4_t horiz_const = vdupq_n_s32((1 << (ROUND0_BITS - 1)) / 2); if (width == 4) { const uint8x16_t perm_tbl = vld1q_u8(kDotProdPermuteTbl); @@ -1165,14 +1161,11 @@ const int8x16_t x_filter = vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8); - const int bd = 8; // This shim of 1 << ((ROUND0_BITS - 1) - 1) enables us to use non-rounding // shifts in convolution kernels - which are generally faster than rounding // shifts on modern CPUs. The outermost -1 is needed because we halved the // filter values. - const int32x4_t horiz_const = vdupq_n_s32((1 << (bd + FILTER_BITS - 2)) + - (1 << ((ROUND0_BITS - 1) - 1))); - const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1)); + const int32x4_t horiz_const = vdupq_n_s32(1 << ((ROUND0_BITS - 1) - 1)); const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { @@ -1203,14 +1196,14 @@ int16x8_t v_s8 = convolve6_8_2d_h(h_s8, x_filter, permute_tbl, horiz_const); - uint8x8_t d0 = convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, - y_filter, vert_const); - uint8x8_t d1 = convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, - y_filter, vert_const); - uint8x8_t d2 = convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7, - y_filter, vert_const); - uint8x8_t d3 = convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8, - y_filter, vert_const); + uint8x8_t d0 = + convolve6_8_2d_v(v_s0, v_s1, v_s2, v_s3, v_s4, v_s5, y_filter); + uint8x8_t d1 = + convolve6_8_2d_v(v_s1, v_s2, v_s3, v_s4, v_s5, v_s6, y_filter); + uint8x8_t d2 = + convolve6_8_2d_v(v_s2, v_s3, v_s4, v_s5, v_s6, v_s7, y_filter); + uint8x8_t d3 = + convolve6_8_2d_v(v_s3, v_s4, v_s5, v_s6, v_s7, v_s8, y_filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3); @@ -1241,13 +1234,10 @@ const int8x16_t x_filter = vcombine_s8(vext_s8(x_filter_s8, x_filter_s8, 1), x_filter_s8); - const int bd = 8; // Adding a shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding // shifts - which are generally faster than rounding shifts on modern CPUs. // Halve the total because we halved the filter values. - const int32x4_t horiz_const = vdupq_n_s32( - ((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))) / 2); - const int16x8_t vert_const = vdupq_n_s16(1 << (bd - 1)); + const int32x4_t horiz_const = vdupq_n_s32((1 << (ROUND0_BITS - 1)) / 2); if (w == 4) { const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); @@ -1278,8 +1268,8 @@ int16x4_t d2 = convolve4_4_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter); int16x4_t d3 = convolve4_4_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter); - uint8x8_t d01 = vqmovun_s16(vsubq_s16(vcombine_s16(d0, d1), vert_const)); - uint8x8_t d23 = vqmovun_s16(vsubq_s16(vcombine_s16(d2, d3), vert_const)); + uint8x8_t d01 = vqmovun_s16(vcombine_s16(d0, d1)); + uint8x8_t d23 = vqmovun_s16(vcombine_s16(d2, d3)); store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01); store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23); @@ -1325,14 +1315,10 @@ int16x8_t v_s6 = convolve6_8_2d_h(h_s6, x_filter, permute_tbl, horiz_const); - uint8x8_t d0 = - convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter, vert_const); - uint8x8_t d1 = - convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter, vert_const); - uint8x8_t d2 = - convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter, vert_const); - uint8x8_t d3 = - convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter, vert_const); + uint8x8_t d0 = convolve4_8_2d_v(v_s0, v_s1, v_s2, v_s3, y_filter); + uint8x8_t d1 = convolve4_8_2d_v(v_s1, v_s2, v_s3, v_s4, y_filter); + uint8x8_t d2 = convolve4_8_2d_v(v_s2, v_s3, v_s4, v_s5, y_filter); + uint8x8_t d3 = convolve4_8_2d_v(v_s3, v_s4, v_s5, v_s6, y_filter); store_u8_8x4(d, dst_stride, d0, d1, d2, d3);
diff --git a/av1/common/arm/convolve_neon_i8mm.h b/av1/common/arm/convolve_neon_i8mm.h index 38e651f..57743a5 100644 --- a/av1/common/arm/convolve_neon_i8mm.h +++ b/av1/common/arm/convolve_neon_i8mm.h
@@ -78,8 +78,6 @@ // The no-op filter should never be used here. assert(x_filter_ptr[5] != 128); - const int bd = 8; - // Split 12-tap filter into two 6-tap filters, masking the top two elements. // { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0 } const int8x8_t mask = vcreate_s8(0x0000ffffffffffff); @@ -97,8 +95,7 @@ // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts // in convolution kernels - which are generally faster than rounding shifts on // modern CPUs. - const int32x4_t horiz_const = - vdupq_n_s32((1 << (bd + FILTER_BITS - 1)) + (1 << (ROUND0_BITS - 1))); + const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1)); if (w <= 4) { const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl);
diff --git a/av1/common/arm/convolve_sve2.c b/av1/common/arm/convolve_sve2.c index ae00386..9e2eb48 100644 --- a/av1/common/arm/convolve_sve2.c +++ b/av1/common/arm/convolve_sve2.c
@@ -60,9 +60,6 @@ // The no-op filter should never be used here. assert(vgetq_lane_s16(y_filter_0_7, 5) != 128); - const int bd = 8; - const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kSVEDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. @@ -129,9 +126,6 @@ vcombine_s16(vqrshrn_n_s32(d2, 2 * FILTER_BITS - ROUND0_BITS), vqrshrn_n_s32(d3, 2 * FILTER_BITS - ROUND0_BITS)); - dd01 = vsubq_s16(dd01, sub_const); - dd23 = vsubq_s16(dd23, sub_const); - uint8x8_t d01 = vqmovun_s16(dd01); uint8x8_t d23 = vqmovun_s16(dd23);