arm/warp_plane_*: Hoist filter load if alpha == beta == 0 In warp affine the common case is that both alpha and beta are zero, in which case the vector of filter values does not need to be re-loaded on each iteration of the loop. LLVM 18 appears to struggle to do this itself, so manually hoist out the vector and pass it as a parameter instead. On a Neoverse V2 machine this seems to give a geomean ~0.9% reduction in times measured for the warp affine speed tests for the cases where alpha and beta are both zero. Change-Id: I1eaffe07e55f8c93f1d4a822321cbe09f8250c99
diff --git a/av1/common/arm/warp_plane_neon.c b/av1/common/arm/warp_plane_neon.c index 2604eaf..3656beb 100644 --- a/av1/common/arm/warp_plane_neon.c +++ b/av1/common/arm/warp_plane_neon.c
@@ -75,13 +75,10 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, - int sx) { +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); - int16x8_t f_s16 = - vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); - int16x8_t in16_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in))); int16x8_t in16_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(in))); @@ -102,12 +99,16 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx) { - const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); - int16x8_t f_s16 = vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); + return horizontal_filter_4x1_f1_beta0(in, f_s16); +} + +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); int16x8_t in16_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(in))); int16x8_t in16_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(in))); @@ -137,6 +138,13 @@ return vreinterpretq_s16_u16(res); } +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, + int sx) { + int16x8_t f_s16 = + vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); + return horizontal_filter_8x1_f1_beta0(in, f_s16); +} + static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res, int sy) { int16x4_t s0 = vget_low_s16(src[0]);
diff --git a/av1/common/arm/warp_plane_neon.h b/av1/common/arm/warp_plane_neon.h index 7fceb3a..777ac4b 100644 --- a/av1/common/arm/warp_plane_neon.h +++ b/av1/common/arm/warp_plane_neon.h
@@ -36,6 +36,12 @@ static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, int sx); +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16); + +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16); + static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res, int sy); @@ -169,7 +175,9 @@ if (p_width == 4) { if (beta == 0) { if (alpha == 0) { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1, sx4); + int16x8_t f_s16 = vld1q_s16( + (int16_t *)(av1_warped_filter + (sx4 >> WARPEDDIFF_PREC_BITS))); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f1_beta0, f_s16); } else { APPLY_HORIZONTAL_SHIFT(horizontal_filter_4x1_f4, sx4, alpha); } @@ -185,7 +193,9 @@ } else { if (beta == 0) { if (alpha == 0) { - APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1, sx4); + int16x8_t f_s16 = vld1q_s16( + (int16_t *)(av1_warped_filter + (sx4 >> WARPEDDIFF_PREC_BITS))); + APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f1_beta0, f_s16); } else { APPLY_HORIZONTAL_SHIFT(horizontal_filter_8x1_f8, sx4, alpha); }
diff --git a/av1/common/arm/warp_plane_neon_i8mm.c b/av1/common/arm/warp_plane_neon_i8mm.c index 45ffaae..7ef7626 100644 --- a/av1/common/arm/warp_plane_neon_i8mm.c +++ b/av1/common/arm/warp_plane_neon_i8mm.c
@@ -84,13 +84,10 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, - int sx) { +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); - int16x8_t f_s16 = - vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); - int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]); @@ -110,12 +107,16 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx) { - const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); - int16x8_t f_s16 = vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); + return horizontal_filter_4x1_f1_beta0(in, f_s16); +} + +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); @@ -142,6 +143,13 @@ return vreinterpretq_s16_u16(res); } +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, + int sx) { + int16x8_t f_s16 = + vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); + return horizontal_filter_8x1_f1_beta0(in, f_s16); +} + static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res, int sy) { int16x4_t s0 = vget_low_s16(src[0]);
diff --git a/av1/common/arm/warp_plane_sve.c b/av1/common/arm/warp_plane_sve.c index 49d23f9..51b1bb7 100644 --- a/av1/common/arm/warp_plane_sve.c +++ b/av1/common/arm/warp_plane_sve.c
@@ -87,13 +87,10 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, - int sx) { +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_4x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); - int16x8_t f_s16 = - vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); - int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); uint8x16_t perm0 = vld1q_u8(&usdot_permute_idx[0]); @@ -113,12 +110,16 @@ return vreinterpretq_s16_u16(res); } -static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, +static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f1(const uint8x16_t in, int sx) { - const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); - int16x8_t f_s16 = vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); + return horizontal_filter_4x1_f1_beta0(in, f_s16); +} + +static AOM_FORCE_INLINE int16x8_t +horizontal_filter_8x1_f1_beta0(const uint8x16_t in, int16x8_t f_s16) { + const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1)); int8x16_t f_s8 = vcombine_s8(vmovn_s16(f_s16), vmovn_s16(f_s16)); @@ -145,6 +146,13 @@ return vreinterpretq_s16_u16(res); } +static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f1(const uint8x16_t in, + int sx) { + int16x8_t f_s16 = + vld1q_s16((int16_t *)(av1_warped_filter + (sx >> WARPEDDIFF_PREC_BITS))); + return horizontal_filter_8x1_f1_beta0(in, f_s16); +} + static AOM_FORCE_INLINE void vertical_filter_4x1_f1(const int16x8_t *src, int32x4_t *res, int sy) { int16x4_t s0 = vget_low_s16(src[0]);