Add Neon Low Bit-depth for SADSkip Performance: Speed Improvement over C 0 3.2x 1 2.5x 2 2.53x 3 1.8x 4 1.78x 5 1.7x Change-Id: I97d27c984295624d7856cdfaab9e25221803b92e
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 5295156..0d6cf35 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -667,26 +667,26 @@ specialize qw/aom_sad16x64 sse2/; specialize qw/aom_sad64x16 sse2/; - specialize qw/aom_sad_skip_128x128 avx2 sse2/; - specialize qw/aom_sad_skip_128x64 avx2 sse2/; - specialize qw/aom_sad_skip_64x128 avx2 sse2/; - specialize qw/aom_sad_skip_64x64 avx2 sse2/; - specialize qw/aom_sad_skip_64x32 avx2 sse2/; - specialize qw/aom_sad_skip_32x64 avx2 sse2/; - specialize qw/aom_sad_skip_32x32 avx2 sse2/; - specialize qw/aom_sad_skip_32x16 avx2 sse2/; - specialize qw/aom_sad_skip_16x32 sse2/; - specialize qw/aom_sad_skip_16x16 sse2/; - specialize qw/aom_sad_skip_16x8 sse2/; - specialize qw/aom_sad_skip_8x16 sse2/; - specialize qw/aom_sad_skip_8x8 sse2/; - specialize qw/aom_sad_skip_4x8 sse2/; + specialize qw/aom_sad_skip_128x128 avx2 sse2 neon/; + specialize qw/aom_sad_skip_128x64 avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x128 avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x64 avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x32 avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x64 avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x32 avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x16 avx2 sse2 neon/; + specialize qw/aom_sad_skip_16x32 sse2 neon/; + specialize qw/aom_sad_skip_16x16 sse2 neon/; + specialize qw/aom_sad_skip_16x8 sse2 neon/; + specialize qw/aom_sad_skip_8x16 sse2 neon/; + specialize qw/aom_sad_skip_8x8 sse2 neon/; + specialize qw/aom_sad_skip_4x8 sse2 neon/; - specialize qw/aom_sad_skip_4x16 sse2/; - specialize qw/aom_sad_skip_8x32 sse2/; - specialize qw/aom_sad_skip_32x8 sse2/; - specialize qw/aom_sad_skip_16x64 sse2/; - specialize qw/aom_sad_skip_64x16 sse2/; + specialize qw/aom_sad_skip_4x16 sse2 neon/; + specialize qw/aom_sad_skip_8x32 sse2 neon/; + specialize qw/aom_sad_skip_32x8 sse2 neon/; + specialize qw/aom_sad_skip_16x64 sse2 neon/; + specialize qw/aom_sad_skip_64x16 sse2 neon/; specialize qw/aom_sad128x128_avg avx2 sse2/; specialize qw/aom_sad128x64_avg avx2 sse2/; @@ -907,30 +907,30 @@ specialize qw/aom_sad32x8x4d sse2/; specialize qw/aom_sad64x16x4d sse2/; - specialize qw/aom_sad_skip_128x128x4d avx2 sse2/; - specialize qw/aom_sad_skip_128x64x4d avx2 sse2/; - specialize qw/aom_sad_skip_64x128x4d avx2 sse2/; - specialize qw/aom_sad_skip_64x64x4d avx2 sse2/; - specialize qw/aom_sad_skip_64x32x4d avx2 sse2/; - specialize qw/aom_sad_skip_64x16x4d avx2 sse2/; - specialize qw/aom_sad_skip_32x64x4d avx2 sse2/; - specialize qw/aom_sad_skip_32x32x4d avx2 sse2/; - specialize qw/aom_sad_skip_32x16x4d avx2 sse2/; - specialize qw/aom_sad_skip_32x8x4d avx2 sse2/; + specialize qw/aom_sad_skip_128x128x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_128x64x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x128x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x64x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x32x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_64x16x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x64x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x32x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x16x4d avx2 sse2 neon/; + specialize qw/aom_sad_skip_32x8x4d avx2 sse2 neon/; - specialize qw/aom_sad_skip_16x64x4d sse2/; - specialize qw/aom_sad_skip_16x32x4d sse2/; - specialize qw/aom_sad_skip_16x16x4d sse2/; - specialize qw/aom_sad_skip_16x8x4d sse2/; - specialize qw/aom_sad_skip_8x16x4d sse2/; - specialize qw/aom_sad_skip_8x8x4d sse2/; - specialize qw/aom_sad_skip_4x16x4d sse2/; - specialize qw/aom_sad_skip_4x8x4d sse2/; - specialize qw/aom_sad_skip_4x32x4d sse2/; - specialize qw/aom_sad_skip_4x16x4d sse2/; - specialize qw/aom_sad_skip_8x32x4d sse2/; - specialize qw/aom_sad_skip_32x8x4d sse2/; - specialize qw/aom_sad_skip_64x16x4d sse2/; + specialize qw/aom_sad_skip_16x64x4d sse2 neon/; + specialize qw/aom_sad_skip_16x32x4d sse2 neon/; + specialize qw/aom_sad_skip_16x16x4d sse2 neon/; + specialize qw/aom_sad_skip_16x8x4d sse2 neon/; + specialize qw/aom_sad_skip_8x16x4d sse2 neon/; + specialize qw/aom_sad_skip_8x8x4d sse2 neon/; + specialize qw/aom_sad_skip_4x16x4d sse2 neon/; + specialize qw/aom_sad_skip_4x8x4d sse2 neon/; + specialize qw/aom_sad_skip_4x32x4d sse2 neon/; + specialize qw/aom_sad_skip_4x16x4d sse2 neon/; + specialize qw/aom_sad_skip_8x32x4d sse2 neon/; + specialize qw/aom_sad_skip_32x8x4d sse2 neon/; + specialize qw/aom_sad_skip_64x16x4d sse2 neon/; specialize qw/aom_sad128x128x4d_avg sse2/; specialize qw/aom_sad128x64x4d_avg sse2/;
diff --git a/aom_dsp/arm/sad4d_neon.c b/aom_dsp/arm/sad4d_neon.c index 606950a..0e633b5 100644 --- a/aom_dsp/arm/sad4d_neon.c +++ b/aom_dsp/arm/sad4d_neon.c
@@ -224,3 +224,368 @@ res[2] = horizontal_long_add_16x8(vec_sum_ref2_lo, vec_sum_ref2_hi); res[3] = horizontal_long_add_16x8(vec_sum_ref3_lo, vec_sum_ref3_hi); } + +static INLINE unsigned int horizontal_add_16x4(const uint16x4_t vec_16x4) { + const uint32x2_t a = vpaddl_u16(vec_16x4); + const uint64x1_t b = vpaddl_u32(a); + return vget_lane_u32(vreinterpret_u32_u64(b), 0); +} + +static INLINE unsigned int horizontal_add_16x8(const uint16x8_t vec_16x8) { + const uint32x4_t a = vpaddlq_u16(vec_16x8); + const uint64x2_t b = vpaddlq_u32(a); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +} + +static void sad_row4_neon(uint16x4_t *vec_src, const uint8x8_t q0, + const uint8x8_t ref) { + uint8x8_t q2 = vabd_u8(q0, ref); + *vec_src = vpadal_u8(*vec_src, q2); +} + +static void sad_row8_neon(uint16x4_t *vec_src, const uint8x8_t *q0, + const uint8_t *ref_ptr) { + uint8x8_t q1 = vld1_u8(ref_ptr); + uint8x8_t q2 = vabd_u8(*q0, q1); + *vec_src = vpadal_u8(*vec_src, q2); +} + +static void sad_row16_neon(uint16x8_t *vec_src, const uint8x16_t *q0, + const uint8_t *ref_ptr) { + uint8x16_t q1 = vld1q_u8(ref_ptr); + uint8x16_t q2 = vabdq_u8(*q0, q1); + *vec_src = vpadalq_u8(*vec_src, q2); +} + +void aom_sadMxNx4d_neon(int width, int height, const uint8_t *src, + int src_stride, const uint8_t *const ref[4], + int ref_stride, uint32_t res[4]) { + const uint8_t *ref0, *ref1, *ref2, *ref3; + + ref0 = ref[0]; + ref1 = ref[1]; + ref2 = ref[2]; + ref3 = ref[3]; + + res[0] = 0; + res[1] = 0; + res[2] = 0; + res[3] = 0; + + switch (width) { + case 4: { + uint32_t src4, ref40, ref41, ref42, ref43; + uint32x2_t q8 = vdup_n_u32(0); + uint32x2_t q4 = vdup_n_u32(0); + uint32x2_t q5 = vdup_n_u32(0); + uint32x2_t q6 = vdup_n_u32(0); + uint32x2_t q7 = vdup_n_u32(0); + + for (int i = 0; i < height / 2; i++) { + uint16x4_t q0 = vdup_n_u16(0); + uint16x4_t q1 = vdup_n_u16(0); + uint16x4_t q2 = vdup_n_u16(0); + uint16x4_t q3 = vdup_n_u16(0); + + memcpy(&src4, src, 4); + memcpy(&ref40, ref0, 4); + memcpy(&ref41, ref1, 4); + memcpy(&ref42, ref2, 4); + memcpy(&ref43, ref3, 4); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + q8 = vset_lane_u32(src4, q8, 0); + q4 = vset_lane_u32(ref40, q4, 0); + q5 = vset_lane_u32(ref41, q5, 0); + q6 = vset_lane_u32(ref42, q6, 0); + q7 = vset_lane_u32(ref43, q7, 0); + + memcpy(&src4, src, 4); + memcpy(&ref40, ref0, 4); + memcpy(&ref41, ref1, 4); + memcpy(&ref42, ref2, 4); + memcpy(&ref43, ref3, 4); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + q8 = vset_lane_u32(src4, q8, 1); + q4 = vset_lane_u32(ref40, q4, 1); + q5 = vset_lane_u32(ref41, q5, 1); + q6 = vset_lane_u32(ref42, q6, 1); + q7 = vset_lane_u32(ref43, q7, 1); + + sad_row4_neon(&q0, vreinterpret_u8_u32(q8), vreinterpret_u8_u32(q4)); + sad_row4_neon(&q1, vreinterpret_u8_u32(q8), vreinterpret_u8_u32(q5)); + sad_row4_neon(&q2, vreinterpret_u8_u32(q8), vreinterpret_u8_u32(q6)); + sad_row4_neon(&q3, vreinterpret_u8_u32(q8), vreinterpret_u8_u32(q7)); + + res[0] += horizontal_add_16x4(q0); + res[1] += horizontal_add_16x4(q1); + res[2] += horizontal_add_16x4(q2); + res[3] += horizontal_add_16x4(q3); + } + break; + } + case 8: { + for (int i = 0; i < height; i++) { + uint16x4_t q0 = vdup_n_u16(0); + uint16x4_t q1 = vdup_n_u16(0); + uint16x4_t q2 = vdup_n_u16(0); + uint16x4_t q3 = vdup_n_u16(0); + + uint8x8_t q5 = vld1_u8(src); + + sad_row8_neon(&q0, &q5, ref0); + sad_row8_neon(&q1, &q5, ref1); + sad_row8_neon(&q2, &q5, ref2); + sad_row8_neon(&q3, &q5, ref3); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + res[0] += horizontal_add_16x4(q0); + res[1] += horizontal_add_16x4(q1); + res[2] += horizontal_add_16x4(q2); + res[3] += horizontal_add_16x4(q3); + } + break; + } + case 16: { + for (int i = 0; i < height; i++) { + uint16x8_t q0 = vdupq_n_u16(0); + uint16x8_t q1 = vdupq_n_u16(0); + uint16x8_t q2 = vdupq_n_u16(0); + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q4 = vld1q_u8(src); + + sad_row16_neon(&q0, &q4, ref0); + sad_row16_neon(&q1, &q4, ref1); + sad_row16_neon(&q2, &q4, ref2); + sad_row16_neon(&q3, &q4, ref3); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + res[0] += horizontal_add_16x8(q0); + res[1] += horizontal_add_16x8(q1); + res[2] += horizontal_add_16x8(q2); + res[3] += horizontal_add_16x8(q3); + } + break; + } + case 32: { + for (int i = 0; i < height; i++) { + uint16x8_t q0 = vdupq_n_u16(0); + uint16x8_t q1 = vdupq_n_u16(0); + uint16x8_t q2 = vdupq_n_u16(0); + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q4 = vld1q_u8(src); + + sad_row16_neon(&q0, &q4, ref0); + sad_row16_neon(&q1, &q4, ref1); + sad_row16_neon(&q2, &q4, ref2); + sad_row16_neon(&q3, &q4, ref3); + + q4 = vld1q_u8(src + 16); + + sad_row16_neon(&q0, &q4, ref0 + 16); + sad_row16_neon(&q1, &q4, ref1 + 16); + sad_row16_neon(&q2, &q4, ref2 + 16); + sad_row16_neon(&q3, &q4, ref3 + 16); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + res[0] += horizontal_add_16x8(q0); + res[1] += horizontal_add_16x8(q1); + res[2] += horizontal_add_16x8(q2); + res[3] += horizontal_add_16x8(q3); + } + break; + } + case 64: { + for (int i = 0; i < height; i++) { + uint16x8_t q0 = vdupq_n_u16(0); + uint16x8_t q1 = vdupq_n_u16(0); + uint16x8_t q2 = vdupq_n_u16(0); + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q4 = vld1q_u8(src); + + sad_row16_neon(&q0, &q4, ref0); + sad_row16_neon(&q1, &q4, ref1); + sad_row16_neon(&q2, &q4, ref2); + sad_row16_neon(&q3, &q4, ref3); + + q4 = vld1q_u8(src + 16); + + sad_row16_neon(&q0, &q4, ref0 + 16); + sad_row16_neon(&q1, &q4, ref1 + 16); + sad_row16_neon(&q2, &q4, ref2 + 16); + sad_row16_neon(&q3, &q4, ref3 + 16); + + q4 = vld1q_u8(src + 32); + + sad_row16_neon(&q0, &q4, ref0 + 32); + sad_row16_neon(&q1, &q4, ref1 + 32); + sad_row16_neon(&q2, &q4, ref2 + 32); + sad_row16_neon(&q3, &q4, ref3 + 32); + + q4 = vld1q_u8(src + 48); + + sad_row16_neon(&q0, &q4, ref0 + 48); + sad_row16_neon(&q1, &q4, ref1 + 48); + sad_row16_neon(&q2, &q4, ref2 + 48); + sad_row16_neon(&q3, &q4, ref3 + 48); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + res[0] += horizontal_add_16x8(q0); + res[1] += horizontal_add_16x8(q1); + res[2] += horizontal_add_16x8(q2); + res[3] += horizontal_add_16x8(q3); + } + break; + } + case 128: { + for (int i = 0; i < height; i++) { + uint16x8_t q0 = vdupq_n_u16(0); + uint16x8_t q1 = vdupq_n_u16(0); + uint16x8_t q2 = vdupq_n_u16(0); + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q4 = vld1q_u8(src); + + sad_row16_neon(&q0, &q4, ref0); + sad_row16_neon(&q1, &q4, ref1); + sad_row16_neon(&q2, &q4, ref2); + sad_row16_neon(&q3, &q4, ref3); + + q4 = vld1q_u8(src + 16); + + sad_row16_neon(&q0, &q4, ref0 + 16); + sad_row16_neon(&q1, &q4, ref1 + 16); + sad_row16_neon(&q2, &q4, ref2 + 16); + sad_row16_neon(&q3, &q4, ref3 + 16); + + q4 = vld1q_u8(src + 32); + + sad_row16_neon(&q0, &q4, ref0 + 32); + sad_row16_neon(&q1, &q4, ref1 + 32); + sad_row16_neon(&q2, &q4, ref2 + 32); + sad_row16_neon(&q3, &q4, ref3 + 32); + + q4 = vld1q_u8(src + 48); + + sad_row16_neon(&q0, &q4, ref0 + 48); + sad_row16_neon(&q1, &q4, ref1 + 48); + sad_row16_neon(&q2, &q4, ref2 + 48); + sad_row16_neon(&q3, &q4, ref3 + 48); + + q4 = vld1q_u8(src + 64); + + sad_row16_neon(&q0, &q4, ref0 + 64); + sad_row16_neon(&q1, &q4, ref1 + 64); + sad_row16_neon(&q2, &q4, ref2 + 64); + sad_row16_neon(&q3, &q4, ref3 + 64); + + q4 = vld1q_u8(src + 80); + + sad_row16_neon(&q0, &q4, ref0 + 80); + sad_row16_neon(&q1, &q4, ref1 + 80); + sad_row16_neon(&q2, &q4, ref2 + 80); + sad_row16_neon(&q3, &q4, ref3 + 80); + + q4 = vld1q_u8(src + 96); + + sad_row16_neon(&q0, &q4, ref0 + 96); + sad_row16_neon(&q1, &q4, ref1 + 96); + sad_row16_neon(&q2, &q4, ref2 + 96); + sad_row16_neon(&q3, &q4, ref3 + 96); + + q4 = vld1q_u8(src + 112); + + sad_row16_neon(&q0, &q4, ref0 + 112); + sad_row16_neon(&q1, &q4, ref1 + 112); + sad_row16_neon(&q2, &q4, ref2 + 112); + sad_row16_neon(&q3, &q4, ref3 + 112); + + src += src_stride; + ref0 += ref_stride; + ref1 += ref_stride; + ref2 += ref_stride; + ref3 += ref_stride; + + res[0] += horizontal_add_16x8(q0); + res[1] += horizontal_add_16x8(q1); + res[2] += horizontal_add_16x8(q2); + res[3] += horizontal_add_16x8(q3); + } + } + } +} + +#define sad_skip_MxN_neon(m, n) \ + void aom_sad_skip_##m##x##n##x4d_neon(const uint8_t *src, int src_stride, \ + const uint8_t *const ref[4], \ + int ref_stride, uint32_t res[4]) { \ + aom_sadMxNx4d_neon(m, ((n) >> 1), src, 2 * src_stride, ref, \ + 2 * ref_stride, res); \ + res[0] <<= 1; \ + res[1] <<= 1; \ + res[2] <<= 1; \ + res[3] <<= 1; \ + } + +sad_skip_MxN_neon(4, 8); +sad_skip_MxN_neon(4, 16); +sad_skip_MxN_neon(4, 32); + +sad_skip_MxN_neon(8, 8); +sad_skip_MxN_neon(8, 16); +sad_skip_MxN_neon(8, 32); + +sad_skip_MxN_neon(16, 8); +sad_skip_MxN_neon(16, 16); +sad_skip_MxN_neon(16, 32); +sad_skip_MxN_neon(16, 64); + +sad_skip_MxN_neon(32, 8); +sad_skip_MxN_neon(32, 16); +sad_skip_MxN_neon(32, 32); +sad_skip_MxN_neon(32, 64); + +sad_skip_MxN_neon(64, 16); +sad_skip_MxN_neon(64, 32); +sad_skip_MxN_neon(64, 64); +sad_skip_MxN_neon(64, 128); + +sad_skip_MxN_neon(128, 64); +sad_skip_MxN_neon(128, 128); +#undef sad_skip_MxN_neon
diff --git a/aom_dsp/arm/sad_neon.c b/aom_dsp/arm/sad_neon.c index 18581f2..17d9994 100644 --- a/aom_dsp/arm/sad_neon.c +++ b/aom_dsp/arm/sad_neon.c
@@ -10,13 +10,12 @@ */ #include <arm_neon.h> - #include "config/aom_config.h" - +#include "config/aom_dsp_rtcd.h" #include "aom/aom_integer.h" -unsigned int aom_sad8x16_neon(unsigned char *src_ptr, int src_stride, - unsigned char *ref_ptr, int ref_stride) { +unsigned int aom_sad8x16_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { uint8x8_t d0, d8; uint16x8_t q12; uint32x4_t q1; @@ -46,8 +45,8 @@ return vget_lane_u32(d5, 0); } -unsigned int aom_sad4x4_neon(unsigned char *src_ptr, int src_stride, - unsigned char *ref_ptr, int ref_stride) { +unsigned int aom_sad4x4_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { uint8x8_t d0, d8; uint16x8_t q12; uint32x2_t d1; @@ -74,8 +73,8 @@ return vget_lane_u32(vreinterpret_u32_u64(d3), 0); } -unsigned int aom_sad16x8_neon(unsigned char *src_ptr, int src_stride, - unsigned char *ref_ptr, int ref_stride) { +unsigned int aom_sad16x8_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride) { uint8x16_t q0, q4; uint16x8_t q12, q13; uint32x4_t q1; @@ -293,3 +292,262 @@ } return horizontal_add_16x8(vec_accum); } + +static INLINE unsigned int sad128xh_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + int sum = 0; + for (int i = 0; i < h; i++) { + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q0 = vld1q_u8(src_ptr); + uint8x16_t q1 = vld1q_u8(ref_ptr); + uint8x16_t q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 16); + q1 = vld1q_u8(ref_ptr + 16); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 32); + q1 = vld1q_u8(ref_ptr + 32); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 48); + q1 = vld1q_u8(ref_ptr + 48); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 64); + q1 = vld1q_u8(ref_ptr + 64); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 80); + q1 = vld1q_u8(ref_ptr + 80); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 96); + q1 = vld1q_u8(ref_ptr + 96); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 112); + q1 = vld1q_u8(ref_ptr + 112); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + src_ptr += src_stride; + ref_ptr += ref_stride; + + sum += horizontal_add_16x8(q3); + } + + return sum; +} + +static INLINE unsigned int sad64xh_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + int sum = 0; + for (int i = 0; i < h; i++) { + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q0 = vld1q_u8(src_ptr); + uint8x16_t q1 = vld1q_u8(ref_ptr); + uint8x16_t q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 16); + q1 = vld1q_u8(ref_ptr + 16); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 32); + q1 = vld1q_u8(ref_ptr + 32); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 48); + q1 = vld1q_u8(ref_ptr + 48); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + src_ptr += src_stride; + ref_ptr += ref_stride; + + sum += horizontal_add_16x8(q3); + } + + return sum; +} + +static INLINE unsigned int sad32xh_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + int sum = 0; + for (int i = 0; i < h; i++) { + uint16x8_t q3 = vdupq_n_u16(0); + + uint8x16_t q0 = vld1q_u8(src_ptr); + uint8x16_t q1 = vld1q_u8(ref_ptr); + uint8x16_t q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + q0 = vld1q_u8(src_ptr + 16); + q1 = vld1q_u8(ref_ptr + 16); + q2 = vabdq_u8(q0, q1); + q3 = vpadalq_u8(q3, q2); + + sum += horizontal_add_16x8(q3); + + src_ptr += src_stride; + ref_ptr += ref_stride; + } + + return sum; +} + +static INLINE unsigned int sad16xh_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + int sum = 0; + for (int i = 0; i < h; i++) { + uint8x8_t q0 = vld1_u8(src_ptr); + uint8x8_t q1 = vld1_u8(ref_ptr); + for (int x = 0; x < 4; x++) + sum += vget_lane_u16(vpaddl_u8(vabd_u8(q0, q1)), x); + q0 = vld1_u8(src_ptr + 8); + q1 = vld1_u8(ref_ptr + 8); + for (int x = 0; x < 4; x++) + sum += vget_lane_u16(vpaddl_u8(vabd_u8(q0, q1)), x); + + src_ptr += src_stride; + ref_ptr += ref_stride; + } + + return sum; +} + +static INLINE unsigned int sad8xh_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + uint16x8_t q3 = vdupq_n_u16(0); + for (int y = 0; y < h; y++) { + uint8x8_t q0 = vld1_u8(src_ptr); + uint8x8_t q1 = vld1_u8(ref_ptr); + src_ptr += src_stride; + ref_ptr += ref_stride; + q3 = vabal_u8(q3, q0, q1); + } + return horizontal_add_16x8(q3); +} + +static INLINE unsigned int sad4xh_neon(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + uint16x8_t q3 = vdupq_n_u16(0); + uint32x2_t q0 = vdup_n_u32(0); + uint32x2_t q1 = vdup_n_u32(0); + uint32_t src4, ref4; + for (int y = 0; y < h / 2; y++) { + memcpy(&src4, src_ptr, 4); + memcpy(&ref4, ref_ptr, 4); + src_ptr += src_stride; + ref_ptr += ref_stride; + q0 = vset_lane_u32(src4, q0, 0); + q1 = vset_lane_u32(ref4, q1, 0); + + memcpy(&src4, src_ptr, 4); + memcpy(&ref4, ref_ptr, 4); + src_ptr += src_stride; + ref_ptr += ref_stride; + q0 = vset_lane_u32(src4, q0, 1); + q1 = vset_lane_u32(ref4, q1, 1); + + q3 = vabal_u8(q3, vreinterpret_u8_u32(q0), vreinterpret_u8_u32(q1)); + } + return horizontal_add_16x8(q3); +} + +#define FSADS128_H(h) \ + unsigned int aom_sad_skip_128x##h##_neon( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + const uint32_t sum = sad128xh_neon(src_ptr, 2 * src_stride, ref_ptr, \ + 2 * ref_stride, h / 2); \ + return 2 * sum; \ + } +FSADS128_H(128); +FSADS128_H(64); +#undef FSADS128_H + +#define FSADS64_H(h) \ + unsigned int aom_sad_skip_64x##h##_neon( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return 2 * sad64xh_neon(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ + h / 2); \ + } + +FSADS64_H(128); +FSADS64_H(64); +FSADS64_H(32); +FSADS64_H(16); +#undef FSADS64_H + +#define FSADS32_H(h) \ + unsigned int aom_sad_skip_32x##h##_neon( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return 2 * sad32xh_neon(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ + h / 2); \ + } + +FSADS32_H(64); +FSADS32_H(32); +FSADS32_H(16); +FSADS32_H(8); +#undef FSADS32_H + +#define FSADS16_H(h) \ + unsigned int aom_sad_skip_16x##h##_neon( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return 2 * sad16xh_neon(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ + h / 2); \ + } + +FSADS16_H(64); +FSADS16_H(32); +FSADS16_H(16); +FSADS16_H(8); +#undef FSADS16_H + +#define FSADS8_H(h) \ + unsigned int aom_sad_skip_8x##h##_neon( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return 2 * sad8xh_neon(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ + h / 2); \ + } + +FSADS8_H(32); +FSADS8_H(16); +FSADS8_H(8); +#undef FSADS8_H + +#define FSADS4_H(h) \ + unsigned int aom_sad_skip_4x##h##_neon( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return 2 * sad4xh_neon(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ + h / 2); \ + } + +FSADS4_H(16); +FSADS4_H(8); +#undef FSADS4_H
diff --git a/test/sad_test.cc b/test/sad_test.cc index fdcf49f..e561859 100644 --- a/test/sad_test.cc +++ b/test/sad_test.cc
@@ -1780,6 +1780,53 @@ make_tuple(16, 16, &aom_sad16x16x4d_neon, -1), }; INSTANTIATE_TEST_SUITE_P(NEON, SADx4Test, ::testing::ValuesIn(x4d_neon_tests)); +const SadSkipMxNParam skip_neon_tests[] = { + make_tuple(128, 128, &aom_sad_skip_128x128_neon, -1), + make_tuple(128, 64, &aom_sad_skip_128x64_neon, -1), + make_tuple(64, 128, &aom_sad_skip_64x128_neon, -1), + make_tuple(64, 64, &aom_sad_skip_64x64_neon, -1), + make_tuple(64, 32, &aom_sad_skip_64x32_neon, -1), + make_tuple(64, 16, &aom_sad_skip_64x16_neon, -1), + make_tuple(32, 64, &aom_sad_skip_32x64_neon, -1), + make_tuple(32, 32, &aom_sad_skip_32x32_neon, -1), + make_tuple(32, 16, &aom_sad_skip_32x16_neon, -1), + make_tuple(32, 8, &aom_sad_skip_32x8_neon, -1), + make_tuple(16, 64, &aom_sad_skip_16x64_neon, -1), + make_tuple(16, 32, &aom_sad_skip_16x32_neon, -1), + make_tuple(16, 16, &aom_sad_skip_16x16_neon, -1), + make_tuple(16, 8, &aom_sad_skip_16x8_neon, -1), + make_tuple(8, 32, &aom_sad_skip_8x32_neon, -1), + make_tuple(8, 16, &aom_sad_skip_8x16_neon, -1), + make_tuple(8, 8, &aom_sad_skip_8x8_neon, -1), + make_tuple(4, 16, &aom_sad_skip_4x16_neon, -1), + make_tuple(4, 8, &aom_sad_skip_4x8_neon, -1), +}; +INSTANTIATE_TEST_SUITE_P(NEON, SADSkipTest, + ::testing::ValuesIn(skip_neon_tests)); + +const SadSkipMxNx4Param skip_x4d_neon_tests[] = { + make_tuple(128, 128, &aom_sad_skip_128x128x4d_neon, -1), + make_tuple(128, 64, &aom_sad_skip_128x64x4d_neon, -1), + make_tuple(64, 128, &aom_sad_skip_64x128x4d_neon, -1), + make_tuple(64, 64, &aom_sad_skip_64x64x4d_neon, -1), + make_tuple(64, 32, &aom_sad_skip_64x32x4d_neon, -1), + make_tuple(64, 16, &aom_sad_skip_64x16x4d_neon, -1), + make_tuple(32, 64, &aom_sad_skip_32x64x4d_neon, -1), + make_tuple(32, 32, &aom_sad_skip_32x32x4d_neon, -1), + make_tuple(32, 16, &aom_sad_skip_32x16x4d_neon, -1), + make_tuple(32, 8, &aom_sad_skip_32x8x4d_neon, -1), + make_tuple(16, 64, &aom_sad_skip_16x64x4d_neon, -1), + make_tuple(16, 32, &aom_sad_skip_16x32x4d_neon, -1), + make_tuple(16, 16, &aom_sad_skip_16x16x4d_neon, -1), + make_tuple(16, 8, &aom_sad_skip_16x8x4d_neon, -1), + make_tuple(8, 8, &aom_sad_skip_8x8x4d_neon, -1), + make_tuple(8, 16, &aom_sad_skip_8x16x4d_neon, -1), + make_tuple(8, 32, &aom_sad_skip_8x32x4d_neon, -1), + make_tuple(4, 8, &aom_sad_skip_4x8x4d_neon, -1), + make_tuple(4, 16, &aom_sad_skip_4x16x4d_neon, -1), +}; +INSTANTIATE_TEST_SUITE_P(NEON, SADSkipx4Test, + ::testing::ValuesIn(skip_x4d_neon_tests)); #endif // HAVE_NEON //------------------------------------------------------------------------------