warp_plane_{neon_i8mm,sve}.c: Move add_const into USDOT
The existing code performs an addition after the dot-product
instructions, however it is possible to instead initialize the
dot-product accumulator with a non-zero value and perform the
accumulation there.
This doesn't actually save an instruction since the compiler must emit a
MOV to set up an accumulator instead of an ADD, however this still gives
a win overall since it shortens the length of the overall dependency
chain.
Benchmarking on a Neoverse V2 machine with LLVM 18 this gives a geomean
0.5% reduction in the times reported by the warp affine speed tests,
including a 2% geomean reduction for the most common alpha=0 beta=0
cases.
Change-Id: Iae5d3b37764628cf4c90dc15a124129b41d2953b
diff --git a/av1/common/arm/warp_plane_neon_i8mm.c b/av1/common/arm/warp_plane_neon_i8mm.c
index 9ccc863..45ffaae 100644
--- a/av1/common/arm/warp_plane_neon_i8mm.c
+++ b/av1/common/arm/warp_plane_neon_i8mm.c
@@ -19,7 +19,10 @@
static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in,
int sx, int alpha) {
- const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+ // Only put the constant in every other lane to avoid double-counting when
+ // performing the pairwise add later.
+ const int32x4_t add_const =
+ vreinterpretq_s32_u64(vdupq_n_u64(1 << (8 + FILTER_BITS - 1)));
// Loading the 8 filter taps
int16x8_t f[4];
@@ -33,21 +36,22 @@
uint8x8_t in2 = vget_low_u8(vextq_u8(in, in, 2));
uint8x8_t in3 = vget_low_u8(vextq_u8(in, in, 3));
- int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
- int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
+ int32x4_t m01 = vusdotq_s32(add_const, vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(add_const, vcombine_u8(in2, in3), f23_u8);
- int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
-
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ int32x4_t m0123 = vpaddq_s32(m01, m23);
uint16x8_t res =
- vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS), vdup_n_u16(0));
return vreinterpretq_s16_u16(res);
}
static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in,
int sx, int alpha) {
- const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+ // Only put the constant in every other lane to avoid double-counting when
+ // performing the pairwise add later.
+ const int32x4_t add_const =
+ vreinterpretq_s32_u64(vdupq_n_u64(1 << (8 + FILTER_BITS - 1)));
// Loading the 8 filter taps
int16x8_t f[8];
@@ -67,19 +71,16 @@
uint8x8_t in6 = vget_low_u8(vextq_u8(in, in, 6));
uint8x8_t in7 = vget_low_u8(vextq_u8(in, in, 7));
- int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
- int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
- int32x4_t m45 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in4, in5), f45_u8);
- int32x4_t m67 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in6, in7), f67_u8);
+ int32x4_t m01 = vusdotq_s32(add_const, vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(add_const, vcombine_u8(in2, in3), f23_u8);
+ int32x4_t m45 = vusdotq_s32(add_const, vcombine_u8(in4, in5), f45_u8);
+ int32x4_t m67 = vusdotq_s32(add_const, vcombine_u8(in6, in7), f67_u8);
- int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
- int32x4_t tmp_res_high = vpaddq_s32(m45, m67);
+ int32x4_t m0123 = vpaddq_s32(m01, m23);
+ int32x4_t m4567 = vpaddq_s32(m45, m67);
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
- tmp_res_high = vaddq_s32(tmp_res_high, add_const);
-
- uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
- vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS),
+ vqrshrun_n_s32(m4567, ROUND0_BITS));
return vreinterpretq_s16_u16(res);
}
@@ -101,15 +102,11 @@
uint8x16_t in_0123 = vqtbl1q_u8(in, perm0);
uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
- int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ int32x4_t m0123 = vusdotq_laneq_s32(add_const, in_0123, f_s8, 0);
m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
- int32x4_t tmp_res_low = m0123;
-
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
-
uint16x8_t res =
- vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS), vdup_n_u16(0));
return vreinterpretq_s16_u16(res);
}
@@ -134,20 +131,14 @@
uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
uint8x16_t in_89ab = vqtbl1q_u8(in, perm2);
- int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ int32x4_t m0123 = vusdotq_laneq_s32(add_const, in_0123, f_s8, 0);
m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
- int32x4_t m4567 = vusdotq_laneq_s32(vdupq_n_s32(0), in_4567, f_s8, 0);
+ int32x4_t m4567 = vusdotq_laneq_s32(add_const, in_4567, f_s8, 0);
m4567 = vusdotq_laneq_s32(m4567, in_89ab, f_s8, 1);
- int32x4_t tmp_res_low = m0123;
- int32x4_t tmp_res_high = m4567;
-
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
- tmp_res_high = vaddq_s32(tmp_res_high, add_const);
-
- uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
- vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS),
+ vqrshrun_n_s32(m4567, ROUND0_BITS));
return vreinterpretq_s16_u16(res);
}
diff --git a/av1/common/arm/warp_plane_sve.c b/av1/common/arm/warp_plane_sve.c
index 9d5761b..49d23f9 100644
--- a/av1/common/arm/warp_plane_sve.c
+++ b/av1/common/arm/warp_plane_sve.c
@@ -22,7 +22,10 @@
static AOM_FORCE_INLINE int16x8_t horizontal_filter_4x1_f4(const uint8x16_t in,
int sx, int alpha) {
- const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+ // Only put the constant in every other lane to avoid double-counting when
+ // performing the pairwise add later.
+ const int32x4_t add_const =
+ vreinterpretq_s32_u64(vdupq_n_u64(1 << (8 + FILTER_BITS - 1)));
// Loading the 8 filter taps
int16x8_t f[4];
@@ -36,21 +39,22 @@
uint8x8_t in2 = vget_low_u8(vextq_u8(in, in, 2));
uint8x8_t in3 = vget_low_u8(vextq_u8(in, in, 3));
- int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
- int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
+ int32x4_t m01 = vusdotq_s32(add_const, vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(add_const, vcombine_u8(in2, in3), f23_u8);
- int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
-
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
+ int32x4_t m0123 = vpaddq_s32(m01, m23);
uint16x8_t res =
- vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS), vdup_n_u16(0));
return vreinterpretq_s16_u16(res);
}
static AOM_FORCE_INLINE int16x8_t horizontal_filter_8x1_f8(const uint8x16_t in,
int sx, int alpha) {
- const int32x4_t add_const = vdupq_n_s32(1 << (8 + FILTER_BITS - 1));
+ // Only put the constant in every other lane to avoid double-counting when
+ // performing the pairwise add later.
+ const int32x4_t add_const =
+ vreinterpretq_s32_u64(vdupq_n_u64(1 << (8 + FILTER_BITS - 1)));
// Loading the 8 filter taps
int16x8_t f[8];
@@ -70,19 +74,16 @@
uint8x8_t in6 = vget_low_u8(vextq_u8(in, in, 6));
uint8x8_t in7 = vget_low_u8(vextq_u8(in, in, 7));
- int32x4_t m01 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in0, in1), f01_u8);
- int32x4_t m23 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in2, in3), f23_u8);
- int32x4_t m45 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in4, in5), f45_u8);
- int32x4_t m67 = vusdotq_s32(vdupq_n_s32(0), vcombine_u8(in6, in7), f67_u8);
+ int32x4_t m01 = vusdotq_s32(add_const, vcombine_u8(in0, in1), f01_u8);
+ int32x4_t m23 = vusdotq_s32(add_const, vcombine_u8(in2, in3), f23_u8);
+ int32x4_t m45 = vusdotq_s32(add_const, vcombine_u8(in4, in5), f45_u8);
+ int32x4_t m67 = vusdotq_s32(add_const, vcombine_u8(in6, in7), f67_u8);
- int32x4_t tmp_res_low = vpaddq_s32(m01, m23);
- int32x4_t tmp_res_high = vpaddq_s32(m45, m67);
+ int32x4_t m0123 = vpaddq_s32(m01, m23);
+ int32x4_t m4567 = vpaddq_s32(m45, m67);
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
- tmp_res_high = vaddq_s32(tmp_res_high, add_const);
-
- uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
- vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS),
+ vqrshrun_n_s32(m4567, ROUND0_BITS));
return vreinterpretq_s16_u16(res);
}
@@ -104,15 +105,11 @@
uint8x16_t in_0123 = vqtbl1q_u8(in, perm0);
uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
- int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ int32x4_t m0123 = vusdotq_laneq_s32(add_const, in_0123, f_s8, 0);
m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
- int32x4_t tmp_res_low = m0123;
-
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
-
uint16x8_t res =
- vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS), vdup_n_u16(0));
+ vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS), vdup_n_u16(0));
return vreinterpretq_s16_u16(res);
}
@@ -137,20 +134,14 @@
uint8x16_t in_4567 = vqtbl1q_u8(in, perm1);
uint8x16_t in_89ab = vqtbl1q_u8(in, perm2);
- int32x4_t m0123 = vusdotq_laneq_s32(vdupq_n_s32(0), in_0123, f_s8, 0);
+ int32x4_t m0123 = vusdotq_laneq_s32(add_const, in_0123, f_s8, 0);
m0123 = vusdotq_laneq_s32(m0123, in_4567, f_s8, 1);
- int32x4_t m4567 = vusdotq_laneq_s32(vdupq_n_s32(0), in_4567, f_s8, 0);
+ int32x4_t m4567 = vusdotq_laneq_s32(add_const, in_4567, f_s8, 0);
m4567 = vusdotq_laneq_s32(m4567, in_89ab, f_s8, 1);
- int32x4_t tmp_res_low = m0123;
- int32x4_t tmp_res_high = m4567;
-
- tmp_res_low = vaddq_s32(tmp_res_low, add_const);
- tmp_res_high = vaddq_s32(tmp_res_high, add_const);
-
- uint16x8_t res = vcombine_u16(vqrshrun_n_s32(tmp_res_low, ROUND0_BITS),
- vqrshrun_n_s32(tmp_res_high, ROUND0_BITS));
+ uint16x8_t res = vcombine_u16(vqrshrun_n_s32(m0123, ROUND0_BITS),
+ vqrshrun_n_s32(m4567, ROUND0_BITS));
return vreinterpretq_s16_u16(res);
}
