highway avx2: sad 64 and above Microbenchmark (ns): Speed up is compared against baseline avx2. Positive means faster, negative means slower. | Block size | baseline avx2 | hwy avx2 | hwy avx2 speedup | |------------|---------------|----------|------------------| | 128x128 | 548 | 450 | 17.88% | | 128x64 | 143 | 152 | -6.29% | | 64x128 | 141 | 135 | 4.26% | | 64x64 | 71.5 | 65.1 | 8.95% | | 64x32 | 40.5 | 31.1 | 23.21% | Change-Id: I3169d8c04287d32913bb4cd56d5077ac88f8f00d
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index 33d9713..9ceb109 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake
@@ -181,6 +181,11 @@ "${AOM_ROOT}/aom_dsp/variance.c" "${AOM_ROOT}/aom_dsp/variance.h") + if(CONFIG_HIGHWAY) + list(APPEND AOM_DSP_ENCODER_SOURCES "${AOM_ROOT}/aom_dsp/reduce_sum_hwy.h" + "${AOM_ROOT}/aom_dsp/sad_hwy.h") + endif() + # Flow estimation library and grain/noise table/model. if(NOT CONFIG_REALTIME_ONLY) list(APPEND AOM_DSP_ENCODER_SOURCES @@ -259,6 +264,11 @@ "${AOM_ROOT}/aom_dsp/x86/blk_sse_sum_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sum_squares_avx2.c") + if(CONFIG_HIGHWAY) + list(APPEND AOM_DSP_ENCODER_INTRIN_AVX2 + "${AOM_ROOT}/aom_dsp/x86/sad_hwy_avx2.cc") + endif() + list(APPEND AOM_DSP_ENCODER_INTRIN_AVX "${AOM_ROOT}/aom_dsp/x86/aom_quantize_avx.c")
diff --git a/aom_dsp/reduce_sum_hwy.h b/aom_dsp/reduce_sum_hwy.h new file mode 100644 index 0000000..9f4c005 --- /dev/null +++ b/aom_dsp/reduce_sum_hwy.h
@@ -0,0 +1,69 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AOM_DSP_REDUCE_SUM_HWY_H_ +#define AOM_AOM_DSP_REDUCE_SUM_HWY_H_ + +#include <type_traits> +#include "third_party/highway/hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); + +namespace { +namespace HWY_NAMESPACE { + +namespace hn = hwy::HWY_NAMESPACE; + +template <size_t NumBlocks> +struct BlockReduceTraits; + +template <> +struct BlockReduceTraits<1> { + template <typename D> + HWY_ATTR HWY_INLINE static hn::VFromD<D> ReduceSum(D d, hn::VFromD<D> v) { + (void)d; + return v; + } +}; + +template <size_t NumBlocks> +struct BlockReduceTraits { + static_assert(NumBlocks > 1, + "Primary template BlockReduceTraits assumes NumBlocks > 1"); + static_assert((NumBlocks & (NumBlocks - 1)) == 0, + "BlockReduceTraits requires NumBlocks to be a power of 2."); + + template <typename D> + HWY_ATTR HWY_INLINE static hn::VFromD<hn::BlockDFromD<D>> ReduceSum( + D d, hn::VFromD<D> v) { + (void)d; + constexpr hn::Half<D> half_d; + auto v_half = hn::Add(hn::LowerHalf(half_d, v), hn::UpperHalf(half_d, v)); + return BlockReduceTraits<NumBlocks / 2>::ReduceSum(half_d, v_half); + } +}; + +// ReduceSum across blocks. +// For example, with a 4-block vector with 16 lanes of uint32_t: +// [a3 b3 c3 d3 a2 b2 c2 d2 a1 b1 c1 d1 a0 b0 c0 d0] +// returns a vector with 4 lanes: +// [a3+a2+a1+a0 b3+b2+b1+b0 c3+c2+c1+c0 d3+d2+d1+d0] +template <typename D> +HWY_ATTR HWY_INLINE hn::Vec<hn::BlockDFromD<D>> BlockReduceSum( + D int_tag, hn::VFromD<D> v) { + return BlockReduceTraits<int_tag.MaxBlocks()>::ReduceSum(int_tag, v); +} + +} // namespace HWY_NAMESPACE +} // namespace + +HWY_AFTER_NAMESPACE(); + +#endif // AOM_AOM_DSP_REDUCE_SUM_HWY_H_
diff --git a/aom_dsp/sad_hwy.h b/aom_dsp/sad_hwy.h new file mode 100644 index 0000000..b142425 --- /dev/null +++ b/aom_dsp/sad_hwy.h
@@ -0,0 +1,77 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#ifndef AOM_AOM_DSP_SAD_HWY_H_ +#define AOM_AOM_DSP_SAD_HWY_H_ + +#include "aom_dsp/reduce_sum_hwy.h" +#include "third_party/highway/hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); + +namespace { +namespace HWY_NAMESPACE { + +namespace hn = hwy::HWY_NAMESPACE; + +template <int BlockWidth> +HWY_MAYBE_UNUSED unsigned int SumOfAbsoluteDiff( + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, + int ref_stride, int h, const uint8_t *second_pred = nullptr) { + constexpr hn::CappedTag<uint8_t, BlockWidth> pixel_tag; + constexpr hn::Repartition<uint64_t, decltype(pixel_tag)> intermediate_sum_tag; + const int vw = hn::Lanes(pixel_tag); + auto sum_sad = hn::Zero(intermediate_sum_tag); + const bool is_sad_avg = second_pred != nullptr; + for (int i = 0; i < h; ++i) { + for (int j = 0; j < BlockWidth; j += vw) { + auto src_vec = hn::LoadU(pixel_tag, &src_ptr[j]); + auto ref_vec = hn::LoadU(pixel_tag, &ref_ptr[j]); + if (is_sad_avg) { + auto sec_pred_vec = hn::LoadU(pixel_tag, &second_pred[j]); + ref_vec = hn::AverageRound(ref_vec, sec_pred_vec); + } + auto sad = hn::SumsOf8AbsDiff(src_vec, ref_vec); + sum_sad = hn::Add(sum_sad, sad); + } + src_ptr += src_stride; + ref_ptr += ref_stride; + if (is_sad_avg) { + second_pred += BlockWidth; + } + } + return static_cast<unsigned int>( + hn::ReduceSum(intermediate_sum_tag, sum_sad)); +} + +} // namespace HWY_NAMESPACE +} // namespace + +#define FSAD(w, h, suffix) \ + extern "C" unsigned int aom_sad##w##x##h##_##suffix( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride); \ + HWY_ATTR unsigned int aom_sad##w##x##h##_##suffix( \ + const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ + int ref_stride) { \ + return HWY_NAMESPACE::SumOfAbsoluteDiff<w>(src_ptr, src_stride, ref_ptr, \ + ref_stride, h); \ + } + +#define FOR_EACH_SAD_BLOCK_SIZE(X, suffix) \ + X(128, 128, suffix) \ + X(128, 64, suffix) \ + X(64, 128, suffix) \ + X(64, 64, suffix) \ + X(64, 32, suffix) + +HWY_AFTER_NAMESPACE(); + +#endif // AOM_AOM_DSP_SAD_HWY_H_
diff --git a/aom_dsp/x86/sad_avx2.c b/aom_dsp/x86/sad_avx2.c index f19ff05..ee02380 100644 --- a/aom_dsp/x86/sad_avx2.c +++ b/aom_dsp/x86/sad_avx2.c
@@ -101,11 +101,17 @@ h / 2); \ } +#if CONFIG_HIGHWAY +#define FSAD64 \ + FSADS64_H(64) \ + FSADS64_H(32) +#else #define FSAD64 \ FSAD64_H(64) \ FSAD64_H(32) \ FSADS64_H(64) \ FSADS64_H(32) +#endif #define FSAD32 \ FSAD32_H(64) \
diff --git a/aom_dsp/x86/sad_hwy_avx2.cc b/aom_dsp/x86/sad_hwy_avx2.cc new file mode 100644 index 0000000..2df2646 --- /dev/null +++ b/aom_dsp/x86/sad_hwy_avx2.cc
@@ -0,0 +1,17 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#define HWY_BASELINE_TARGETS HWY_AVX2 +#define HWY_BROKEN_32BIT 0 + +#include "aom_dsp/sad_hwy.h" + +FOR_EACH_SAD_BLOCK_SIZE(FSAD, avx2)
diff --git a/aom_dsp/x86/sad_impl_avx2.c b/aom_dsp/x86/sad_impl_avx2.c index 0d1b5ab..2c6fa24 100644 --- a/aom_dsp/x86/sad_impl_avx2.c +++ b/aom_dsp/x86/sad_impl_avx2.c
@@ -56,6 +56,7 @@ return sum; } +#if !CONFIG_HIGHWAY unsigned int aom_sad128x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride) { unsigned int half_width = 64; @@ -83,6 +84,7 @@ sum += aom_sad128x64_avx2(src_ptr, src_stride, ref_ptr, ref_stride); return sum; } +#endif unsigned int aom_sad_skip_128x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride) {