Highway: add avx2 for SAD With ~2% speed up on 64xh block sizes and 10-11% on 128xh block sizes for SAD and SAD_SKIP Similar performance on SAD_4D, SAD_3D and SAD_4K_SKIP Also add test coverage for highway code on Jenkins thru running avx2 Change-Id: Ic1babd1ddc1f358faf61539df765041e1fac877d
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index b41bc57..3047bc7 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake
@@ -269,6 +269,8 @@ "${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx2.cc") list(APPEND AOM_DSP_ENCODER_INTRIN_AVX512 "${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx512.cc") + list(REMOVE_ITEM AOM_DSP_ENCODER_INTRIN_AVX2 + "${AOM_ROOT}/aom_dsp/x86/sad_impl_avx2.c") endif() list(APPEND AOM_DSP_ENCODER_INTRIN_AVX
diff --git a/aom_dsp/x86/sad4d_avx2.c b/aom_dsp/x86/sad4d_avx2.c index 0c1a03c..8f75f97 100644 --- a/aom_dsp/x86/sad4d_avx2.c +++ b/aom_dsp/x86/sad4d_avx2.c
@@ -146,12 +146,14 @@ SADMXN_AVX2(32, 32) SADMXN_AVX2(32, 64) +#if !CONFIG_HIGHWAY SADMXN_AVX2(64, 32) SADMXN_AVX2(64, 64) SADMXN_AVX2(64, 128) SADMXN_AVX2(128, 64) SADMXN_AVX2(128, 128) +#endif #if !CONFIG_REALTIME_ONLY SADMXN_AVX2(32, 8) @@ -174,12 +176,14 @@ SAD_SKIP_MXN_AVX2(32, 32) SAD_SKIP_MXN_AVX2(32, 64) +#if !CONFIG_HIGHWAY SAD_SKIP_MXN_AVX2(64, 32) SAD_SKIP_MXN_AVX2(64, 64) SAD_SKIP_MXN_AVX2(64, 128) SAD_SKIP_MXN_AVX2(128, 64) SAD_SKIP_MXN_AVX2(128, 128) +#endif #if !CONFIG_REALTIME_ONLY SAD_SKIP_MXN_AVX2(64, 16)
diff --git a/aom_dsp/x86/sad_avx2.c b/aom_dsp/x86/sad_avx2.c index ee02380..fa20a41 100644 --- a/aom_dsp/x86/sad_avx2.c +++ b/aom_dsp/x86/sad_avx2.c
@@ -14,6 +14,8 @@ #include "aom_ports/mem.h" +// SAD, SAD_SKIP and SAD_AVG for 64xh blocks +#if !CONFIG_HIGHWAY static inline unsigned int sad64xh_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, int h) { @@ -42,37 +44,6 @@ return res; } -static inline unsigned int sad32xh_avx2(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride, - int h) { - int i; - __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; - __m256i sum_sad = _mm256_setzero_si256(); - __m256i sum_sad_h; - __m128i sum_sad128; - int ref2_stride = ref_stride << 1; - int src2_stride = src_stride << 1; - int max = h >> 1; - for (i = 0; i < max; i++) { - ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); - ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); - sad1_reg = - _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); - sad2_reg = _mm256_sad_epu8( - ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); - sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); - ref_ptr += ref2_stride; - src_ptr += src2_stride; - } - sum_sad_h = _mm256_srli_si256(sum_sad, 8); - sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); - sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); - sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); - unsigned int res = (unsigned int)_mm_cvtsi128_si32(sum_sad128); - _mm256_zeroupper(); - return res; -} - #define FSAD64_H(h) \ unsigned int aom_sad64x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ @@ -87,49 +58,18 @@ h / 2); \ } -#define FSAD32_H(h) \ - unsigned int aom_sad32x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *ref_ptr, int ref_stride) { \ - return sad32xh_avx2(src_ptr, src_stride, ref_ptr, ref_stride, h); \ - } - -#define FSADS32_H(h) \ - unsigned int aom_sad_skip_32x##h##_avx2( \ - const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ - int ref_stride) { \ - return 2 * sad32xh_avx2(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ - h / 2); \ - } - -#if CONFIG_HIGHWAY -#define FSAD64 \ - FSADS64_H(64) \ - FSADS64_H(32) -#else #define FSAD64 \ FSAD64_H(64) \ FSAD64_H(32) \ FSADS64_H(64) \ FSADS64_H(32) -#endif - -#define FSAD32 \ - FSAD32_H(64) \ - FSAD32_H(32) \ - FSAD32_H(16) \ - FSADS32_H(64) \ - FSADS32_H(32) \ - FSADS32_H(16) /* clang-format off */ FSAD64 -FSAD32 /* clang-format on */ #undef FSAD64 -#undef FSAD32 #undef FSAD64_H -#undef FSAD32_H #define FSADAVG64_H(h) \ unsigned int aom_sad64x##h##_avg_avx2( \ @@ -166,6 +106,79 @@ return res; \ } +#define FSADAVG64 \ + FSADAVG64_H(64) \ + FSADAVG64_H(32) + +/* clang-format off */ +FSADAVG64 +/* clang-format on */ + +#undef FSADAVG64 +#undef FSADAVG64_H +#endif // !CONFIG_HIGHWAY + +// SAD, SAD_SKIP and SAD_AVG for 32xh blocks +static inline unsigned int sad32xh_avx2(const uint8_t *src_ptr, int src_stride, + const uint8_t *ref_ptr, int ref_stride, + int h) { + int i; + __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; + __m256i sum_sad = _mm256_setzero_si256(); + __m256i sum_sad_h; + __m128i sum_sad128; + int ref2_stride = ref_stride << 1; + int src2_stride = src_stride << 1; + int max = h >> 1; + for (i = 0; i < max; i++) { + ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); + ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); + sad1_reg = + _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); + sad2_reg = _mm256_sad_epu8( + ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); + sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); + ref_ptr += ref2_stride; + src_ptr += src2_stride; + } + sum_sad_h = _mm256_srli_si256(sum_sad, 8); + sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); + sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); + sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); + unsigned int res = (unsigned int)_mm_cvtsi128_si32(sum_sad128); + _mm256_zeroupper(); + return res; +} + +#define FSAD32_H(h) \ + unsigned int aom_sad32x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ + const uint8_t *ref_ptr, int ref_stride) { \ + return sad32xh_avx2(src_ptr, src_stride, ref_ptr, ref_stride, h); \ + } + +#define FSADS32_H(h) \ + unsigned int aom_sad_skip_32x##h##_avx2( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return 2 * sad32xh_avx2(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ + h / 2); \ + } + +#define FSAD32 \ + FSAD32_H(64) \ + FSAD32_H(32) \ + FSAD32_H(16) \ + FSADS32_H(64) \ + FSADS32_H(32) \ + FSADS32_H(16) + +/* clang-format off */ +FSAD32 +/* clang-format on */ + +#undef FSAD32 +#undef FSAD32_H + #define FSADAVG32_H(h) \ unsigned int aom_sad32x##h##_avg_avx2( \ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ @@ -205,21 +218,14 @@ return res; \ } -#define FSADAVG64 \ - FSADAVG64_H(64) \ - FSADAVG64_H(32) - #define FSADAVG32 \ FSADAVG32_H(64) \ FSADAVG32_H(32) \ FSADAVG32_H(16) /* clang-format off */ -FSADAVG64 FSADAVG32 /* clang-format on */ -#undef FSADAVG64 #undef FSADAVG32 -#undef FSADAVG64_H #undef FSADAVG32_H
diff --git a/aom_dsp/x86/sad_hwy_avx2.cc b/aom_dsp/x86/sad_hwy_avx2.cc index 2df2646..68462f2 100644 --- a/aom_dsp/x86/sad_hwy_avx2.cc +++ b/aom_dsp/x86/sad_hwy_avx2.cc
@@ -14,4 +14,12 @@ #include "aom_dsp/sad_hwy.h" +// FSAD_4D, FSAD_3D and FSAD_4D_SKIP have similar performance, but migrating +// to highway adds test coverage and simplifies avx2 when it's enabled by +// default. FOR_EACH_SAD_BLOCK_SIZE(FSAD, avx2) +FOR_EACH_SAD_BLOCK_SIZE(FSAD_SKIP, avx2) +FOR_EACH_SAD_BLOCK_SIZE(FSAD_AVG, avx2) +FOR_EACH_SAD_BLOCK_SIZE(FSAD_4D, avx2) +FOR_EACH_SAD_BLOCK_SIZE(FSAD_4D_SKIP, avx2) +FOR_EACH_SAD_BLOCK_SIZE(FSAD_3D, avx2)
diff --git a/aom_dsp/x86/sad_impl_avx2.c b/aom_dsp/x86/sad_impl_avx2.c index 2c6fa24..0d1b5ab 100644 --- a/aom_dsp/x86/sad_impl_avx2.c +++ b/aom_dsp/x86/sad_impl_avx2.c
@@ -56,7 +56,6 @@ return sum; } -#if !CONFIG_HIGHWAY unsigned int aom_sad128x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride) { unsigned int half_width = 64; @@ -84,7 +83,6 @@ sum += aom_sad128x64_avx2(src_ptr, src_stride, ref_ptr, ref_stride); return sum; } -#endif unsigned int aom_sad_skip_128x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride) {