Remove interleaving load/stores in cfl_predict_neon Interleaving is not necessary and interleaving load/stores can be very slow. Write new helper functions for old compiler versions that don't support these _x2/_x4 variants. Change-Id: I9d865acc68528e3ea192d56e2cceab5bad87d14b
diff --git a/aom_dsp/arm/mem_neon.h b/aom_dsp/arm/mem_neon.h index 4ea6881..9cdafec 100644 --- a/aom_dsp/arm/mem_neon.h +++ b/aom_dsp/arm/mem_neon.h
@@ -55,12 +55,52 @@ return res; } +static inline int16x8x2_t vld1q_s16_x2(const int16_t *ptr) { + int16x8x2_t res = { { vld1q_s16(ptr + 0 * 8), vld1q_s16(ptr + 1 * 8) } }; + return res; +} + +static inline int16x8x4_t vld1q_s16_x4(const int16_t *ptr) { + int16x8x4_t res = { { vld1q_s16(ptr + 0 * 8), vld1q_s16(ptr + 1 * 8), + vld1q_s16(ptr + 2 * 8), vld1q_s16(ptr + 3 * 8) } }; + return res; +} + +static inline void vst1_u8_x2(uint8_t *ptr, uint8x8x2_t a) { + vst1_u8(ptr + 0 * 8, a.val[0]); + vst1_u8(ptr + 1 * 8, a.val[1]); +} + +static inline void vst1_u8_x4(uint8_t *ptr, uint8x8x4_t a) { + vst1_u8(ptr + 0 * 8, a.val[0]); + vst1_u8(ptr + 1 * 8, a.val[1]); + vst1_u8(ptr + 2 * 8, a.val[2]); + vst1_u8(ptr + 3 * 8, a.val[3]); +} + +static inline void vst1q_u16_x2(uint16_t *ptr, uint16x8x2_t a) { + vst1q_u16(ptr + 0 * 8, a.val[0]); + vst1q_u16(ptr + 1 * 8, a.val[1]); +} + +static inline void vst1q_u16_x4(uint16_t *ptr, uint16x8x4_t a) { + vst1q_u16(ptr + 0 * 8, a.val[0]); + vst1q_u16(ptr + 1 * 8, a.val[1]); + vst1q_u16(ptr + 2 * 8, a.val[2]); + vst1q_u16(ptr + 3 * 8, a.val[3]); +} + #elif defined(__GNUC__) && !defined(__clang__) // GCC 64-bit. #if __GNUC__ < 8 static inline uint8x16x2_t vld1q_u8_x2(const uint8_t *ptr) { uint8x16x2_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16) } }; return res; } + +static inline int16x8x2_t vld1q_s16_x2(const int16_t *ptr) { + int16x8x2_t res = { { vld1q_s16(ptr + 0 * 8), vld1q_s16(ptr + 1 * 8) } }; + return res; +} #endif // __GNUC__ < 8 #if __GNUC__ < 9 @@ -71,13 +111,30 @@ } #endif // __GNUC__ < 9 -// vld1q_u16_x4 is defined from GCC 8.5.0 and onwards. #if ((__GNUC__ << 8) | __GNUC_MINOR__) < 0x805 static inline uint16x8x4_t vld1q_u16_x4(const uint16_t *ptr) { uint16x8x4_t res = { { vld1q_u16(ptr + 0 * 8), vld1q_u16(ptr + 1 * 8), vld1q_u16(ptr + 2 * 8), vld1q_u16(ptr + 3 * 8) } }; return res; } + +static inline int16x8x4_t vld1q_s16_x4(const int16_t *ptr) { + int16x8x4_t res = { { vld1q_s16(ptr + 0 * 8), vld1q_s16(ptr + 1 * 8), + vld1q_s16(ptr + 2 * 8), vld1q_s16(ptr + 3 * 8) } }; + return res; +} + +static inline void vst1_u8_x2(uint8_t *ptr, uint8x8x2_t a) { + vst1_u8(ptr + 0 * 8, a.val[0]); + vst1_u8(ptr + 1 * 8, a.val[1]); +} + +static inline void vst1_u8_x4(uint8_t *ptr, uint8x8x4_t a) { + vst1_u8(ptr + 0 * 8, a.val[0]); + vst1_u8(ptr + 1 * 8, a.val[1]); + vst1_u8(ptr + 2 * 8, a.val[2]); + vst1_u8(ptr + 3 * 8, a.val[3]); +} #endif // ((__GNUC__ << 8) | __GNUC_MINOR__) < 0x805 #endif // defined(__GNUC__) && !defined(__clang__)
diff --git a/av1/common/arm/cfl_neon.c b/av1/common/arm/cfl_neon.c index c1763ff..e7f0ff0 100644 --- a/av1/common/arm/cfl_neon.c +++ b/av1/common/arm/cfl_neon.c
@@ -13,6 +13,7 @@ #include "config/aom_config.h" #include "config/av1_rtcd.h" +#include "aom_dsp/arm/mem_neon.h" #include "av1/common/cfl.h" static inline void vldsubstq_s16(int16_t *dst, const uint16_t *src, int offset, @@ -428,10 +429,7 @@ static inline int16x8x2_t predict_w16(const int16_t *pred_buf_q3, int16x8_t alpha_sign, int abs_alpha_q12, int16x8_t dc) { - // vld2q_s16 interleaves, which is not useful for prediction. vst1q_s16_x2 - // does not interleave, but is not currently available in the compilier used - // by the AOM build system. - const int16x8x2_t ac_q3 = vld2q_s16(pred_buf_q3); + const int16x8x2_t ac_q3 = vld1q_s16_x2(pred_buf_q3); const int16x8_t ac_sign_0 = veorq_s16(alpha_sign, ac_q3.val[0]); const int16x8_t ac_sign_1 = veorq_s16(alpha_sign, ac_q3.val[1]); const int16x8_t scaled_luma_0 = @@ -447,10 +445,7 @@ static inline int16x8x4_t predict_w32(const int16_t *pred_buf_q3, int16x8_t alpha_sign, int abs_alpha_q12, int16x8_t dc) { - // vld4q_s16 interleaves, which is not useful for prediction. vst1q_s16_x4 - // does not interleave, but is not currently available in the compilier used - // by the AOM build system. - const int16x8x4_t ac_q3 = vld4q_s16(pred_buf_q3); + const int16x8x4_t ac_q3 = vld1q_s16_x4(pred_buf_q3); const int16x8_t ac_sign_0 = veorq_s16(alpha_sign, ac_q3.val[0]); const int16x8_t ac_sign_1 = veorq_s16(alpha_sign, ac_q3.val[1]); const int16x8_t ac_sign_2 = veorq_s16(alpha_sign, ac_q3.val[2]); @@ -497,7 +492,7 @@ predict_w16(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); const uint8x8x2_t predun = { { vqmovun_s16(pred.val[0]), vqmovun_s16(pred.val[1]) } }; - vst2_u8(dst, predun); + vst1_u8_x2(dst, predun); } else { const int16x8x4_t pred = predict_w32(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); @@ -505,7 +500,7 @@ { vqmovun_s16(pred.val[0]), vqmovun_s16(pred.val[1]), vqmovun_s16(pred.val[2]), vqmovun_s16(pred.val[3]) } }; - vst4_u8(dst, predun); + vst1_u8_x4(dst, predun); } dst += dst_stride; } while ((pred_buf_q3 += CFL_BUF_LINE) < end); @@ -574,11 +569,11 @@ } else if (width == 16) { const int16x8x2_t pred = predict_w16(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vst2q_u16(dst, clamp2q_s16(pred, max_16x8)); + vst1q_u16_x2(dst, clamp2q_s16(pred, max_16x8)); } else { const int16x8x4_t pred = predict_w32(pred_buf_q3, alpha_sign, abs_alpha_q12, dc); - vst4q_u16(dst, clamp4q_s16(pred, max_16x8)); + vst1q_u16_x4(dst, clamp4q_s16(pred, max_16x8)); } dst += dst_stride; } while ((pred_buf_q3 += CFL_BUF_LINE) < end);