Add Neon implementation of OBMC SAD functions Add Neon implementation of aom_obmc_sad<w>x<h> functions, as well as the corresponding tests. Change-Id: I0691444aa0250d52ce52e912c082c7770e2903b0
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index f9dbc55..6e11237 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake
@@ -265,6 +265,7 @@ "${AOM_ROOT}/aom_dsp/arm/hadamard_neon.c" "${AOM_ROOT}/aom_dsp/arm/avg_neon.c" "${AOM_ROOT}/aom_dsp/arm/obmc_variance_neon.c" + "${AOM_ROOT}/aom_dsp/arm/obmc_sad_neon.c" "${AOM_ROOT}/aom_dsp/arm/sse_neon.c" "${AOM_ROOT}/aom_dsp/arm/sum_squares_neon.c")
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index e02d244..81b3153 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -968,7 +968,7 @@ ($w, $h) = @$_; add_proto qw/unsigned int/, "aom_obmc_sad${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask"; if (! (($w == 128 && $h == 32) || ($w == 32 && $h == 128))) { - specialize "aom_obmc_sad${w}x${h}", qw/sse4_1 avx2/; + specialize "aom_obmc_sad${w}x${h}", qw/sse4_1 avx2 neon/; } }
diff --git a/aom_dsp/arm/obmc_sad_neon.c b/aom_dsp/arm/obmc_sad_neon.c new file mode 100644 index 0000000..60656e5 --- /dev/null +++ b/aom_dsp/arm/obmc_sad_neon.c
@@ -0,0 +1,250 @@ +/* + * Copyright (c) 2023, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" +#include "mem_neon.h" +#include "sum_neon.h" + +static INLINE void obmc_sad_8x1_s16_neon(int16x8_t ref_s16, const int32_t *mask, + const int32_t *wsrc, uint32x4_t *sum) { + int32x4_t wsrc_lo = vld1q_s32(wsrc); + int32x4_t wsrc_hi = vld1q_s32(wsrc + 4); + + int32x4_t mask_lo = vld1q_s32(mask); + int32x4_t mask_hi = vld1q_s32(mask + 4); + + int16x8_t mask_s16 = + vuzpq_s16(vreinterpretq_s16_s32(mask_lo), vreinterpretq_s16_s32(mask_hi)) + .val[0]; + + int32x4_t pre_lo = vmull_s16(vget_low_s16(ref_s16), vget_low_s16(mask_s16)); + int32x4_t pre_hi = vmull_s16(vget_high_s16(ref_s16), vget_high_s16(mask_s16)); + + uint32x4_t abs_lo = vreinterpretq_u32_s32(vabdq_s32(wsrc_lo, pre_lo)); + uint32x4_t abs_hi = vreinterpretq_u32_s32(vabdq_s32(wsrc_hi, pre_hi)); + + *sum = vrsraq_n_u32(*sum, abs_lo, 12); + *sum = vrsraq_n_u32(*sum, abs_hi, 12); +} + +#if defined(__aarch64__) + +// Use tbl for doing a double-width zero extension from 8->32 bits since we can +// do this in one instruction rather than two (indices out of range (255 here) +// are set to zero by tbl). +DECLARE_ALIGNED(16, static const uint8_t, obmc_variance_permute_idx[]) = { + 0, 255, 255, 255, 1, 255, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255, + 4, 255, 255, 255, 5, 255, 255, 255, 6, 255, 255, 255, 7, 255, 255, 255, + 8, 255, 255, 255, 9, 255, 255, 255, 10, 255, 255, 255, 11, 255, 255, 255, + 12, 255, 255, 255, 13, 255, 255, 255, 14, 255, 255, 255, 15, 255, 255, 255 +}; + +static INLINE void obmc_sad_8x1_s32_neon(uint32x4_t ref_u32_lo, + uint32x4_t ref_u32_hi, + const int32_t *mask, + const int32_t *wsrc, + uint32x4_t sum[2]) { + int32x4_t wsrc_lo = vld1q_s32(wsrc); + int32x4_t wsrc_hi = vld1q_s32(wsrc + 4); + int32x4_t mask_lo = vld1q_s32(mask); + int32x4_t mask_hi = vld1q_s32(mask + 4); + + int32x4_t pre_lo = vmulq_s32(vreinterpretq_s32_u32(ref_u32_lo), mask_lo); + int32x4_t pre_hi = vmulq_s32(vreinterpretq_s32_u32(ref_u32_hi), mask_hi); + + uint32x4_t abs_lo = vreinterpretq_u32_s32(vabdq_s32(wsrc_lo, pre_lo)); + uint32x4_t abs_hi = vreinterpretq_u32_s32(vabdq_s32(wsrc_hi, pre_hi)); + + sum[0] = vrsraq_n_u32(sum[0], abs_lo, 12); + sum[1] = vrsraq_n_u32(sum[1], abs_hi, 12); +} + +static INLINE unsigned int obmc_sad_large_neon(const uint8_t *ref, + int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int width, + int height) { + uint32x4_t sum[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; + + // Use tbl for doing a double-width zero extension from 8->32 bits since we + // can do this in one instruction rather than two. + uint8x16_t pre_idx0 = vld1q_u8(&obmc_variance_permute_idx[0]); + uint8x16_t pre_idx1 = vld1q_u8(&obmc_variance_permute_idx[16]); + uint8x16_t pre_idx2 = vld1q_u8(&obmc_variance_permute_idx[32]); + uint8x16_t pre_idx3 = vld1q_u8(&obmc_variance_permute_idx[48]); + + int h = height; + do { + int w = width; + const uint8_t *ref_ptr = ref; + do { + uint8x16_t r = vld1q_u8(ref_ptr); + + uint32x4_t ref_u32_lo = vreinterpretq_u32_u8(vqtbl1q_u8(r, pre_idx0)); + uint32x4_t ref_u32_hi = vreinterpretq_u32_u8(vqtbl1q_u8(r, pre_idx1)); + obmc_sad_8x1_s32_neon(ref_u32_lo, ref_u32_hi, mask, wsrc, sum); + + ref_u32_lo = vreinterpretq_u32_u8(vqtbl1q_u8(r, pre_idx2)); + ref_u32_hi = vreinterpretq_u32_u8(vqtbl1q_u8(r, pre_idx3)); + obmc_sad_8x1_s32_neon(ref_u32_lo, ref_u32_hi, mask + 8, wsrc + 8, sum); + + ref_ptr += 16; + wsrc += 16; + mask += 16; + w -= 16; + } while (w != 0); + + ref += ref_stride; + } while (--h != 0); + + return horizontal_add_u32x4(vaddq_u32(sum[0], sum[1])); +} + +#else // !defined(__aarch64__) + +static INLINE unsigned int obmc_sad_large_neon(const uint8_t *ref, + int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int width, + int height) { + uint32x4_t sum = vdupq_n_u32(0); + + int h = height; + do { + int w = width; + const uint8_t *ref_ptr = ref; + do { + uint8x16_t r = vld1q_u8(ref_ptr); + + int16x8_t ref_s16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(r))); + obmc_sad_8x1_s16_neon(ref_s16, mask, wsrc, &sum); + + ref_s16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(r))); + obmc_sad_8x1_s16_neon(ref_s16, mask + 8, wsrc + 8, &sum); + + ref_ptr += 16; + wsrc += 16; + mask += 16; + w -= 16; + } while (w != 0); + + ref += ref_stride; + } while (--h != 0); + + return horizontal_add_u32x4(sum); +} + +#endif // defined(__aarch64__) + +static INLINE unsigned int obmc_sad_128xh_neon(const uint8_t *ref, + int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int h) { + return obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 128, h); +} + +static INLINE unsigned int obmc_sad_64xh_neon(const uint8_t *ref, + int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int h) { + return obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 64, h); +} + +static INLINE unsigned int obmc_sad_32xh_neon(const uint8_t *ref, + int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int h) { + return obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 32, h); +} + +static INLINE unsigned int obmc_sad_16xh_neon(const uint8_t *ref, + int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int h) { + return obmc_sad_large_neon(ref, ref_stride, wsrc, mask, 16, h); +} + +static INLINE unsigned int obmc_sad_8xh_neon(const uint8_t *ref, int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int height) { + uint32x4_t sum = vdupq_n_u32(0); + + int h = height; + do { + uint8x8_t r = vld1_u8(ref); + + int16x8_t ref_s16 = vreinterpretq_s16_u16(vmovl_u8(r)); + obmc_sad_8x1_s16_neon(ref_s16, mask, wsrc, &sum); + + ref += ref_stride; + wsrc += 8; + mask += 8; + } while (--h != 0); + + return horizontal_add_u32x4(sum); +} + +static INLINE unsigned int obmc_sad_4xh_neon(const uint8_t *ref, int ref_stride, + const int32_t *wsrc, + const int32_t *mask, int height) { + uint32x4_t sum = vdupq_n_u32(0); + + int h = height / 2; + do { + uint8x8_t r = load_unaligned_u8(ref, ref_stride); + + int16x8_t ref_s16 = vreinterpretq_s16_u16(vmovl_u8(r)); + obmc_sad_8x1_s16_neon(ref_s16, mask, wsrc, &sum); + + ref += 2 * ref_stride; + wsrc += 8; + mask += 8; + } while (--h != 0); + + return horizontal_add_u32x4(sum); +} + +#define OBMC_SAD_WXH_NEON(w, h) \ + unsigned int aom_obmc_sad##w##x##h##_neon( \ + const uint8_t *ref, int ref_stride, const int32_t *wsrc, \ + const int32_t *mask) { \ + return obmc_sad_##w##xh_neon(ref, ref_stride, wsrc, mask, h); \ + } + +OBMC_SAD_WXH_NEON(4, 4) +OBMC_SAD_WXH_NEON(4, 8) +OBMC_SAD_WXH_NEON(4, 16) + +OBMC_SAD_WXH_NEON(8, 4) +OBMC_SAD_WXH_NEON(8, 8) +OBMC_SAD_WXH_NEON(8, 16) +OBMC_SAD_WXH_NEON(8, 32) + +OBMC_SAD_WXH_NEON(16, 4) +OBMC_SAD_WXH_NEON(16, 8) +OBMC_SAD_WXH_NEON(16, 16) +OBMC_SAD_WXH_NEON(16, 32) +OBMC_SAD_WXH_NEON(16, 64) + +OBMC_SAD_WXH_NEON(32, 8) +OBMC_SAD_WXH_NEON(32, 16) +OBMC_SAD_WXH_NEON(32, 32) +OBMC_SAD_WXH_NEON(32, 64) + +OBMC_SAD_WXH_NEON(64, 16) +OBMC_SAD_WXH_NEON(64, 32) +OBMC_SAD_WXH_NEON(64, 64) +OBMC_SAD_WXH_NEON(64, 128) + +OBMC_SAD_WXH_NEON(128, 64) +OBMC_SAD_WXH_NEON(128, 128)
diff --git a/test/obmc_sad_test.cc b/test/obmc_sad_test.cc index 9b70366..8d13ac1 100644 --- a/test/obmc_sad_test.cc +++ b/test/obmc_sad_test.cc
@@ -147,6 +147,37 @@ ::testing::ValuesIn(avx2_functions)); #endif // HAVE_AVX2 +#if HAVE_NEON +const ObmcSadTest::ParamType neon_functions[] = { + TestFuncs(aom_obmc_sad128x128_c, aom_obmc_sad128x128_neon), + TestFuncs(aom_obmc_sad128x64_c, aom_obmc_sad128x64_neon), + TestFuncs(aom_obmc_sad64x128_c, aom_obmc_sad64x128_neon), + TestFuncs(aom_obmc_sad64x64_c, aom_obmc_sad64x64_neon), + TestFuncs(aom_obmc_sad64x32_c, aom_obmc_sad64x32_neon), + TestFuncs(aom_obmc_sad32x64_c, aom_obmc_sad32x64_neon), + TestFuncs(aom_obmc_sad32x32_c, aom_obmc_sad32x32_neon), + TestFuncs(aom_obmc_sad32x16_c, aom_obmc_sad32x16_neon), + TestFuncs(aom_obmc_sad16x32_c, aom_obmc_sad16x32_neon), + TestFuncs(aom_obmc_sad16x16_c, aom_obmc_sad16x16_neon), + TestFuncs(aom_obmc_sad16x8_c, aom_obmc_sad16x8_neon), + TestFuncs(aom_obmc_sad8x16_c, aom_obmc_sad8x16_neon), + TestFuncs(aom_obmc_sad8x8_c, aom_obmc_sad8x8_neon), + TestFuncs(aom_obmc_sad8x4_c, aom_obmc_sad8x4_neon), + TestFuncs(aom_obmc_sad4x8_c, aom_obmc_sad4x8_neon), + TestFuncs(aom_obmc_sad4x4_c, aom_obmc_sad4x4_neon), + + TestFuncs(aom_obmc_sad64x16_c, aom_obmc_sad64x16_neon), + TestFuncs(aom_obmc_sad16x64_c, aom_obmc_sad16x64_neon), + TestFuncs(aom_obmc_sad32x8_c, aom_obmc_sad32x8_neon), + TestFuncs(aom_obmc_sad8x32_c, aom_obmc_sad8x32_neon), + TestFuncs(aom_obmc_sad16x4_c, aom_obmc_sad16x4_neon), + TestFuncs(aom_obmc_sad4x16_c, aom_obmc_sad4x16_neon), +}; + +INSTANTIATE_TEST_SUITE_P(NEON, ObmcSadTest, + ::testing::ValuesIn(neon_functions)); +#endif // HAVE_NEON + #if CONFIG_AV1_HIGHBITDEPTH //////////////////////////////////////////////////////////////////////////////// // High bit-depth