| /* |
| * Copyright (c) 2025, Alliance for Open Media. All rights reserved. |
| * |
| * This source code is subject to the terms of the BSD 2 Clause License and |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| * was not distributed with this source code in the LICENSE file, you can |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| * Media Patent License 1.0 was not distributed with this source code in the |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| */ |
| |
| #ifndef AOM_AV1_COMMON_WARP_PLANE_HWY_H_ |
| #define AOM_AV1_COMMON_WARP_PLANE_HWY_H_ |
| |
| #include "av1/common/warped_motion.h" |
| #include "config/av1_rtcd.h" |
| #include "third_party/highway/hwy/highway.h" |
| |
| HWY_BEFORE_NAMESPACE(); |
| |
| namespace { |
| namespace HWY_NAMESPACE { |
| |
| namespace hn = hwy::HWY_NAMESPACE; |
| |
| constexpr hn::ScalableTag<uint8_t> uint8xN_tag; |
| constexpr hn::ScalableTag<uint16_t> uint16xN_tag; |
| |
| constexpr hn::ScalableTag<int16_t> int16xN_tag; |
| constexpr hn::ScalableTag<int32_t> int32xN_tag; |
| constexpr hn::ScalableTag<int64_t> int64xN_tag; |
| |
| constexpr hn::CappedTag<uint8_t, 32> uint8x32_tag; |
| constexpr hn::CappedTag<int16_t, 16> int16x16_tag; |
| |
| constexpr hn::FixedTag<uint8_t, 4> uint8x4_tag; |
| constexpr hn::FixedTag<uint8_t, 8> uint8x8_tag; |
| constexpr hn::FixedTag<uint8_t, 16> uint8x16_tag; |
| constexpr hn::FixedTag<uint16_t, 4> uint16x4_tag; |
| constexpr hn::FixedTag<uint16_t, 8> uint16x8_tag; |
| |
| constexpr hn::FixedTag<int8_t, 8> int8x8_tag; |
| constexpr hn::FixedTag<int8_t, 16> int8x16_tag; |
| constexpr hn::FixedTag<int16_t, 8> int16x8_tag; |
| constexpr hn::FixedTag<int32_t, 4> int32x4_tag; |
| constexpr hn::FixedTag<int64_t, 2> int64x2_tag; |
| |
| constexpr hn::ScalableTag<int8_t> coeff_tag; |
| |
| HWY_ATTR HWY_INLINE hn::Vec<decltype(uint8xN_tag)> GetShuffle0() { |
| return hn::Dup128VecFromValues( |
| uint8xN_tag, 0, 2, 2, 4, 4, 6, 6, 8, 1, 3, 3, 5, 5, 7, 7, 9 // |
| ); |
| } |
| HWY_ATTR HWY_INLINE hn::Vec<decltype(uint8xN_tag)> GetShuffle1() { |
| return hn::Dup128VecFromValues( |
| uint8xN_tag, 4, 6, 6, 8, 8, 10, 10, 12, 5, 7, 7, 9, 9, 11, 11, 13 // |
| ); |
| } |
| HWY_ATTR HWY_INLINE hn::Vec<decltype(uint8xN_tag)> GetShuffle2() { |
| return hn::Dup128VecFromValues( |
| uint8xN_tag, 1, 3, 3, 5, 5, 7, 7, 9, 2, 4, 4, 6, 6, 8, 8, 10 // |
| ); |
| } |
| HWY_ATTR HWY_INLINE hn::Vec<decltype(uint8xN_tag)> GetShuffle3() { |
| return hn::Dup128VecFromValues( |
| uint8xN_tag, 5, 7, 7, 9, 9, 11, 11, 13, 6, 8, 8, 10, 10, 12, 12, 14 // |
| ); |
| } |
| |
| using IVec16 = hn::Vec<decltype(int16xN_tag)>; |
| using IVec32 = hn::Vec<decltype(int32xN_tag)>; |
| using IVec8x16 = hn::Vec<decltype(int8x16_tag)>; |
| |
| template <typename D> |
| HWY_ATTR inline void FilterPixelsHorizontal(D tag, const hn::VFromD<D> src, |
| int16_t *HWY_RESTRICT horz_out, |
| int8_t *HWY_RESTRICT coeff, |
| const IVec16 round_const, |
| const int shift, int row) { |
| constexpr hn::Repartition<int8_t, D> int8_tag; |
| constexpr hn::Repartition<int16_t, D> result_tag; |
| constexpr hn::Repartition<uint16_t, D> unsigned_result_tag; |
| // N.B. coeffs are stored to support the maximum vector width, which may not |
| // be the vector width being filtered on now. |
| const auto coeff0 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 0); |
| const auto coeff1 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 1); |
| const auto coeff2 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 2); |
| const auto coeff3 = hn::Load(int8_tag, coeff + hn::MaxLanes(coeff_tag) * 3); |
| |
| const auto src_0 = |
| hn::TableLookupBytes(src, hn::ResizeBitCast(tag, GetShuffle0())); |
| const auto src_1 = |
| hn::TableLookupBytes(src, hn::ResizeBitCast(tag, GetShuffle1())); |
| const auto src_2 = |
| hn::TableLookupBytes(src, hn::ResizeBitCast(tag, GetShuffle2())); |
| const auto src_3 = |
| hn::TableLookupBytes(src, hn::ResizeBitCast(tag, GetShuffle3())); |
| |
| const auto res_02 = hn::SatWidenMulPairwiseAdd(result_tag, src_0, coeff0); |
| const auto res_46 = hn::SatWidenMulPairwiseAdd(result_tag, src_1, coeff1); |
| const auto res_13 = hn::SatWidenMulPairwiseAdd(result_tag, src_2, coeff2); |
| const auto res_57 = hn::SatWidenMulPairwiseAdd(result_tag, src_3, coeff3); |
| |
| const auto res_even = hn::Add(res_02, res_46); |
| const auto res_odd = hn::Add(res_13, res_57); |
| |
| const auto res = hn::Add(hn::Add(res_even, res_odd), |
| hn::ResizeBitCast(result_tag, round_const)); |
| |
| hn::Store(hn::BitCast(result_tag, |
| hn::ShiftRightSame( |
| hn::BitCast(unsigned_result_tag, res), shift)), |
| result_tag, horz_out + row * hn::MaxLanes(int16x8_tag)); |
| } |
| |
| HWY_ATTR HWY_INLINE IVec8x16 LoadAV1Filter8Bit(unsigned int offset) { |
| const int8_t *HWY_RESTRICT filter_ptr = &av1_filter_8bit[0][0]; |
| return hn::LoadN(int8x16_tag, |
| filter_ptr + (offset >> WARPEDDIFF_PREC_BITS) * 8, 8); |
| } |
| |
| template <typename D> |
| HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAV1Filter8BitLower(D int8_tag, |
| unsigned int offset) { |
| const int8_t *HWY_RESTRICT filter_ptr = &av1_filter_8bit[0][0]; |
| return hn::LoadN(int8_tag, filter_ptr + (offset >> WARPEDDIFF_PREC_BITS) * 8, |
| 8); |
| } |
| |
| template <int Block, typename D> |
| HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAV1Filter8BitUpper(D int8_tag, |
| unsigned int offset, |
| hn::VFromD<D> src) { |
| (void)int8_tag; |
| const int8_t *HWY_RESTRICT filter_ptr = &av1_filter_8bit[0][0]; |
| return hn::InsertBlock<Block>( |
| src, hn::LoadN(int8x16_tag, |
| filter_ptr + (offset >> WARPEDDIFF_PREC_BITS) * 8, 8)); |
| } |
| |
| template <typename D> |
| HWY_ATTR inline void PrepareHorizontalFilterCoefficients( |
| D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { |
| constexpr auto int8_tag = hn::Repartition<int8_t, D>(); |
| constexpr auto int32_tag = hn::Repartition<int32_t, D>(); |
| constexpr auto int64_tag = hn::Repartition<int64_t, D>(); |
| |
| auto tmp_0 = LoadAV1Filter8BitLower(int8_tag, sx + 0 * alpha); |
| auto tmp_1 = LoadAV1Filter8BitLower(int8_tag, sx + 1 * alpha); |
| auto tmp_2 = LoadAV1Filter8BitLower(int8_tag, sx + 2 * alpha); |
| auto tmp_3 = LoadAV1Filter8BitLower(int8_tag, sx + 3 * alpha); |
| auto tmp_4 = LoadAV1Filter8BitLower(int8_tag, sx + 4 * alpha); |
| auto tmp_5 = LoadAV1Filter8BitLower(int8_tag, sx + 5 * alpha); |
| auto tmp_6 = LoadAV1Filter8BitLower(int8_tag, sx + 6 * alpha); |
| auto tmp_7 = LoadAV1Filter8BitLower(int8_tag, sx + 7 * alpha); |
| |
| if constexpr (int16_tag.MaxBlocks() >= 2) { |
| tmp_0 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 0 * alpha, tmp_0); |
| tmp_1 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 1 * alpha, tmp_1); |
| tmp_2 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 2 * alpha, tmp_2); |
| tmp_3 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 3 * alpha, tmp_3); |
| tmp_4 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 4 * alpha, tmp_4); |
| tmp_5 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 5 * alpha, tmp_5); |
| tmp_6 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 6 * alpha, tmp_6); |
| tmp_7 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta + 7 * alpha, tmp_7); |
| } |
| |
| if constexpr (int16_tag.MaxBlocks() >= 3) { |
| tmp_0 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 0 * alpha, tmp_0); |
| tmp_1 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 1 * alpha, tmp_1); |
| tmp_2 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 2 * alpha, tmp_2); |
| tmp_3 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 3 * alpha, tmp_3); |
| tmp_4 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 4 * alpha, tmp_4); |
| tmp_5 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 5 * alpha, tmp_5); |
| tmp_6 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 6 * alpha, tmp_6); |
| tmp_7 = |
| LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2 + 7 * alpha, tmp_7); |
| |
| tmp_0 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 0 * alpha, tmp_0); |
| tmp_1 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 1 * alpha, tmp_1); |
| tmp_2 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 2 * alpha, tmp_2); |
| tmp_3 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 3 * alpha, tmp_3); |
| tmp_4 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 4 * alpha, tmp_4); |
| tmp_5 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 5 * alpha, tmp_5); |
| tmp_6 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 6 * alpha, tmp_6); |
| tmp_7 = |
| LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3 + 7 * alpha, tmp_7); |
| } |
| |
| const auto tmp_0_16 = hn::BitCast(int16_tag, tmp_0); |
| const auto tmp_1_16 = hn::BitCast(int16_tag, tmp_1); |
| const auto tmp_2_16 = hn::BitCast(int16_tag, tmp_2); |
| const auto tmp_3_16 = hn::BitCast(int16_tag, tmp_3); |
| const auto tmp_4_16 = hn::BitCast(int16_tag, tmp_4); |
| const auto tmp_5_16 = hn::BitCast(int16_tag, tmp_5); |
| const auto tmp_6_16 = hn::BitCast(int16_tag, tmp_6); |
| const auto tmp_7_16 = hn::BitCast(int16_tag, tmp_7); |
| |
| const auto tmp_12 = hn::ZipLower(int32_tag, tmp_0_16, tmp_2_16); |
| const auto tmp_13 = hn::ZipLower(int32_tag, tmp_1_16, tmp_3_16); |
| const auto tmp_14 = hn::ZipLower(int32_tag, tmp_4_16, tmp_6_16); |
| const auto tmp_15 = hn::ZipLower(int32_tag, tmp_5_16, tmp_7_16); |
| |
| const auto res_0 = hn::ZipLower(int64_tag, tmp_12, tmp_14); |
| const auto res_1 = hn::ZipUpper(int64_tag, tmp_12, tmp_14); |
| const auto res_2 = hn::ZipLower(int64_tag, tmp_13, tmp_15); |
| const auto res_3 = hn::ZipUpper(int64_tag, tmp_13, tmp_15); |
| |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_0, res_2)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 0); |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_0, res_2)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 1); |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_1, res_3)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 2); |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_1, res_3)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 3); |
| } |
| |
| template <typename D> |
| HWY_ATTR inline void PrepareHorizontalFilterCoefficientsBeta0( |
| D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { |
| (void)int16_tag; |
| (void)beta; |
| const auto tmp_0 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 0 * alpha)); |
| const auto tmp_1 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 1 * alpha)); |
| const auto tmp_2 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 2 * alpha)); |
| const auto tmp_3 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 3 * alpha)); |
| const auto tmp_4 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 4 * alpha)); |
| const auto tmp_5 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 5 * alpha)); |
| const auto tmp_6 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 6 * alpha)); |
| const auto tmp_7 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 7 * alpha)); |
| |
| const auto tmp_02 = hn::ZipLower(int32x4_tag, tmp_0, tmp_2); |
| const auto tmp_13 = hn::ZipLower(int32x4_tag, tmp_1, tmp_3); |
| const auto tmp_46 = hn::ZipLower(int32x4_tag, tmp_4, tmp_6); |
| const auto tmp_57 = hn::ZipLower(int32x4_tag, tmp_5, tmp_7); |
| |
| constexpr auto int8_tag = hn::Repartition<int8_t, D>(); |
| constexpr auto int32_tag = hn::Repartition<int32_t, D>(); |
| constexpr auto int64_tag = hn::Repartition<int64_t, D>(); |
| |
| const auto broadcast_12 = |
| hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_02)); |
| const auto broadcast_13 = |
| hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_13)); |
| const auto broadcast_14 = |
| hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_46)); |
| const auto broadcast_15 = |
| hn::BroadcastBlock<0>(hn::ResizeBitCast(int32_tag, tmp_57)); |
| |
| const auto res_0 = hn::ZipLower(int64_tag, broadcast_12, broadcast_14); |
| const auto res_1 = hn::ZipUpper(int64_tag, broadcast_12, broadcast_14); |
| const auto res_2 = hn::ZipLower(int64_tag, broadcast_13, broadcast_15); |
| const auto res_3 = hn::ZipUpper(int64_tag, broadcast_13, broadcast_15); |
| |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_0, res_2)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 0); |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_0, res_2)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 1); |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveLower(int64_tag, res_1, res_3)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 2); |
| hn::Store(hn::BitCast(int8_tag, hn::InterleaveUpper(int64_tag, res_1, res_3)), |
| int8_tag, coeff + hn::MaxLanes(coeff_tag) * 3); |
| } |
| |
| template <typename D> |
| HWY_ATTR inline void PrepareHorizontalFilterCoefficientsAlpha0( |
| D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { |
| (void)alpha; |
| constexpr auto int8_tag = hn::Repartition<int8_t, D>(); |
| auto tmp_0 = LoadAV1Filter8BitLower(int8_tag, sx); |
| if constexpr (int16_tag.MaxBlocks() >= 2) { |
| tmp_0 = LoadAV1Filter8BitUpper<1>(int8_tag, sx + beta, tmp_0); |
| } |
| if constexpr (int16_tag.MaxBlocks() >= 3) { |
| tmp_0 = LoadAV1Filter8BitUpper<2>(int8_tag, sx + beta * 2, tmp_0); |
| tmp_0 = LoadAV1Filter8BitUpper<3>(int8_tag, sx + beta * 3, tmp_0); |
| } |
| const auto res_0 = hn::BitCast(int16_tag, tmp_0); |
| |
| hn::Store(hn::BitCast(int8_tag, hn::Broadcast<0>(res_0)), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 0); |
| hn::Store(hn::BitCast(int8_tag, hn::Broadcast<1>(res_0)), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 1); |
| hn::Store(hn::BitCast(int8_tag, hn::Broadcast<2>(res_0)), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 2); |
| hn::Store(hn::BitCast(int8_tag, hn::Broadcast<3>(res_0)), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 3); |
| } |
| |
| template <typename D> |
| HWY_ATTR inline void HorizontalFilter(D tag, const hn::VFromD<D> src, |
| int16_t *HWY_RESTRICT horz_out, int sx, |
| int alpha, int beta, int row, |
| const IVec16 round_const, |
| const int reduce_bits_horiz) { |
| HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; |
| PrepareHorizontalFilterCoefficients(hn::Repartition<int16_t, D>(), alpha, |
| beta, sx, coeff); |
| FilterPixelsHorizontal(tag, src, horz_out, coeff, round_const, |
| reduce_bits_horiz, row); |
| } |
| |
| template <typename D> |
| HWY_ATTR inline void PrepareLastHorizontalFilterCoefficients( |
| D int16_tag, int alpha, int beta, int sx, int8_t *HWY_RESTRICT coeff) { |
| (void)int16_tag; |
| (void)beta; |
| const auto tmp_0 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 0 * alpha)); |
| const auto tmp_1 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 1 * alpha)); |
| const auto tmp_2 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 2 * alpha)); |
| const auto tmp_3 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 3 * alpha)); |
| const auto tmp_4 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 4 * alpha)); |
| const auto tmp_5 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 5 * alpha)); |
| const auto tmp_6 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 6 * alpha)); |
| const auto tmp_7 = |
| hn::BitCast(int16x8_tag, LoadAV1Filter8Bit(sx + 7 * alpha)); |
| |
| const auto tmp_8 = hn::ZipLower(int32x4_tag, tmp_0, tmp_2); |
| const auto tmp_9 = hn::ZipLower(int32x4_tag, tmp_1, tmp_3); |
| const auto tmp_10 = hn::ZipLower(int32x4_tag, tmp_4, tmp_6); |
| const auto tmp_11 = hn::ZipLower(int32x4_tag, tmp_5, tmp_7); |
| |
| const auto tmp_12 = hn::ZipLower(int64x2_tag, tmp_8, tmp_10); |
| const auto tmp_13 = hn::ZipUpper(int64x2_tag, tmp_8, tmp_10); |
| const auto tmp_14 = hn::ZipLower(int64x2_tag, tmp_9, tmp_11); |
| const auto tmp_15 = hn::ZipUpper(int64x2_tag, tmp_9, tmp_11); |
| |
| const auto tmp_16 = hn::InterleaveLower(int64x2_tag, tmp_12, tmp_14); |
| const auto tmp_17 = hn::InterleaveUpper(int64x2_tag, tmp_12, tmp_14); |
| const auto tmp_18 = hn::InterleaveLower(int64x2_tag, tmp_13, tmp_15); |
| const auto tmp_19 = hn::InterleaveUpper(int64x2_tag, tmp_13, tmp_15); |
| |
| constexpr auto int8_tag = hn::Repartition<int8_t, D>(); |
| |
| const auto tmp_20 = hn::ResizeBitCast(int8_tag, tmp_16); |
| const auto tmp_21 = hn::ResizeBitCast(int8_tag, tmp_17); |
| const auto tmp_22 = hn::ResizeBitCast(int8_tag, tmp_18); |
| const auto tmp_23 = hn::ResizeBitCast(int8_tag, tmp_19); |
| |
| hn::Store(hn::BroadcastBlock<0>(tmp_20), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 0); |
| hn::Store(hn::BroadcastBlock<0>(tmp_21), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 1); |
| hn::Store(hn::BroadcastBlock<0>(tmp_22), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 2); |
| hn::Store(hn::BroadcastBlock<0>(tmp_23), int8_tag, |
| coeff + hn::MaxLanes(coeff_tag) * 3); |
| } |
| |
| template <typename D> |
| HWY_ATTR HWY_INLINE hn::VFromD<D> LoadRowsClamped( |
| D tag, const uint8_t *HWY_RESTRICT ref, const int stride, const int iy, |
| const int height) { |
| constexpr hn::BlockDFromD<D> block_tag; |
| const int iy0 = clamp(iy + 0, 0, height - 1); |
| auto src = hn::ResizeBitCast(tag, hn::LoadU(block_tag, ref + iy0 * stride)); |
| if constexpr (tag.MaxBlocks() >= 2) { |
| const int iy1 = clamp(iy + 1, 0, height - 1); |
| const auto src_1 = hn::LoadU(block_tag, ref + iy1 * stride); |
| src = hn::InsertBlock<1>(src, src_1); |
| } |
| if constexpr (tag.MaxBlocks() >= 3) { |
| const int iy2 = clamp(iy + 2, 0, height - 1); |
| const auto src_2 = hn::LoadU(block_tag, ref + iy2 * stride); |
| const int iy3 = clamp(iy + 3, 0, height - 1); |
| const auto src_3 = hn::LoadU(block_tag, ref + iy3 * stride); |
| src = hn::InsertBlock<2>(src, src_2); |
| src = hn::InsertBlock<3>(src, src_3); |
| } |
| return src; |
| } |
| |
| template <typename D> |
| HWY_ATTR HWY_INLINE hn::VFromD<D> LoadRowsInterior( |
| D tag, const uint8_t *HWY_RESTRICT ptr, const int stride, const int stride2, |
| const int stride3) { |
| constexpr hn::BlockDFromD<D> block_tag; |
| auto src = hn::ResizeBitCast(tag, hn::LoadU(block_tag, ptr)); |
| if constexpr (tag.MaxBlocks() >= 2) { |
| const auto src_1 = hn::LoadU(block_tag, ptr + stride); |
| src = hn::InsertBlock<1>(src, src_1); |
| } |
| if constexpr (tag.MaxBlocks() >= 3) { |
| const auto src_2 = hn::LoadU(block_tag, ptr + stride2); |
| const auto src_3 = hn::LoadU(block_tag, ptr + stride3); |
| src = hn::InsertBlock<2>(src, src_2); |
| src = hn::InsertBlock<3>(src, src_3); |
| } |
| return src; |
| } |
| |
| enum class HorizontalFilterCoeffs { |
| kAlpha0, |
| kBeta0, |
| kDefault, |
| }; |
| |
| template <bool IsLast, HorizontalFilterCoeffs Filter, typename D> |
| HWY_ATTR void WarpHorizontalPrepareCoeffs(int alpha, int beta, int sx, |
| int8_t *HWY_RESTRICT coeffs) { |
| D int16_tag; |
| switch (Filter) { |
| case HorizontalFilterCoeffs::kAlpha0: |
| PrepareHorizontalFilterCoefficientsAlpha0(int16_tag, alpha, beta, sx, |
| coeffs); |
| return; |
| case HorizontalFilterCoeffs::kBeta0: |
| PrepareHorizontalFilterCoefficientsBeta0(int16_tag, alpha, beta, sx, |
| coeffs); |
| return; |
| case HorizontalFilterCoeffs::kDefault: |
| default: |
| if (IsLast) { |
| PrepareLastHorizontalFilterCoefficients(int16_tag, alpha, beta, sx, |
| coeffs); |
| } else { |
| PrepareHorizontalFilterCoefficients(int16_tag, alpha, beta, sx, coeffs); |
| } |
| return; |
| } |
| } |
| |
| template <bool InnerCoeffUpdate, bool IsLast, HorizontalFilterCoeffs Filter, |
| typename D, typename D16> |
| HWY_ATTR int WarpHorizontalFilterLoop( |
| D tag, D16, const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, |
| int stride, int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, |
| int p_height, int height, int i, const IVec16 round_const, |
| const int reduce_bits_horiz, int k, int8_t *HWY_RESTRICT coeff) { |
| constexpr int kNumRows = tag.MaxBlocks(); |
| for (; k < AOMMIN(8, p_height - i) - kNumRows; k += kNumRows) { |
| const auto src = |
| LoadRowsClamped(tag, ref + ix4 - 7, stride, iy4 + k, height); |
| if constexpr (InnerCoeffUpdate) { |
| int sx = sx4 + beta * (k + 4); |
| WarpHorizontalPrepareCoeffs<IsLast, Filter, D16>(alpha, beta, sx, coeff); |
| } |
| FilterPixelsHorizontal(tag, src, horz_out, coeff, round_const, |
| reduce_bits_horiz, k + 7); |
| } |
| return k; |
| } |
| |
| template <bool InnerCoeffUpdate, bool IsLast, HorizontalFilterCoeffs Filter, |
| typename D, typename D16> |
| HWY_ATTR int WarpHorizontalFilterLoopInterior( |
| D tag, D16, const uint8_t *HWY_RESTRICT base_ptr, |
| int16_t *HWY_RESTRICT horz_out, int stride, int stride2, int stride3, |
| int32_t sx4, int alpha, int beta, int p_height, int i, |
| const IVec16 round_const, const int reduce_bits_horiz, int k, |
| int8_t *HWY_RESTRICT coeff) { |
| constexpr int kNumRows = tag.MaxBlocks(); |
| for (; k < AOMMIN(8, p_height - i) - kNumRows; k += kNumRows) { |
| const auto src = LoadRowsInterior(tag, base_ptr + (k + 7) * stride, stride, |
| stride2, stride3); |
| if constexpr (InnerCoeffUpdate) { |
| int sx = sx4 + beta * (k + 4); |
| WarpHorizontalPrepareCoeffs<IsLast, Filter, D16>(alpha, beta, sx, coeff); |
| } |
| FilterPixelsHorizontal(tag, src, horz_out, coeff, round_const, |
| reduce_bits_horiz, k + 7); |
| } |
| return k; |
| } |
| |
| template <bool InnerCoeffUpdate, HorizontalFilterCoeffs Filter> |
| HWY_ATTR inline void WarpHorizontalFilterTemplate( |
| const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, int stride, |
| int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, |
| int height, int i, const IVec16 round_const, const int reduce_bits_horiz) { |
| int k = -7, iy; |
| HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; |
| if constexpr (!InnerCoeffUpdate) { |
| WarpHorizontalPrepareCoeffs<false, Filter, decltype(int16xN_tag)>( |
| alpha, beta, sx4, coeff); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() >= 3) { |
| k = WarpHorizontalFilterLoop<InnerCoeffUpdate, false, Filter>( |
| uint8xN_tag, int16xN_tag, ref, horz_out, stride, ix4, iy4, sx4, alpha, |
| beta, p_height, height, i, round_const, reduce_bits_horiz, k, coeff); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() >= 2) { |
| k = WarpHorizontalFilterLoop<InnerCoeffUpdate, false, Filter>( |
| uint8x32_tag, int16x16_tag, ref, horz_out, stride, ix4, iy4, sx4, alpha, |
| beta, p_height, height, i, round_const, reduce_bits_horiz, k, coeff); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() == 1) { |
| k = WarpHorizontalFilterLoop<InnerCoeffUpdate, true, Filter>( |
| uint8x16_tag, int16x8_tag, ref, horz_out, stride, ix4, iy4, sx4, alpha, |
| beta, p_height, height, i, round_const, reduce_bits_horiz, k, coeff); |
| } |
| iy = iy4 + k; |
| iy = clamp(iy, 0, height - 1); |
| const auto src = hn::LoadU(uint8x16_tag, ref + iy * stride + ix4 - 7); |
| if constexpr (InnerCoeffUpdate) { |
| int sx = sx4 + beta * (k + 4); |
| WarpHorizontalPrepareCoeffs<true, Filter, decltype(int16x8_tag)>( |
| alpha, beta, sx, coeff); |
| } |
| FilterPixelsHorizontal(uint8x16_tag, src, horz_out, coeff, round_const, |
| reduce_bits_horiz, k + 7); |
| } |
| |
| template <bool InnerCoeffUpdate, HorizontalFilterCoeffs Filter> |
| HWY_ATTR inline void WarpHorizontalFilterTemplateInterior( |
| const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, int stride, |
| int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, |
| int i, const IVec16 round_const, const int reduce_bits_horiz) { |
| const uint8_t *HWY_RESTRICT base_ptr = ref + (iy4 - 7) * stride + ix4 - 7; |
| const int stride2 = stride * 2; |
| const int stride3 = stride * 3; |
| |
| int k = -7; |
| HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; |
| if constexpr (!InnerCoeffUpdate) { |
| WarpHorizontalPrepareCoeffs<false, Filter, decltype(int16xN_tag)>( |
| alpha, beta, sx4, coeff); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() >= 3) { |
| k = WarpHorizontalFilterLoopInterior<InnerCoeffUpdate, false, Filter>( |
| uint8xN_tag, int16xN_tag, base_ptr, horz_out, stride, stride2, stride3, |
| sx4, alpha, beta, p_height, i, round_const, reduce_bits_horiz, k, |
| coeff); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() >= 2) { |
| k = WarpHorizontalFilterLoopInterior<InnerCoeffUpdate, false, Filter>( |
| uint8x32_tag, int16x16_tag, base_ptr, horz_out, stride, stride2, |
| stride3, sx4, alpha, beta, p_height, i, round_const, reduce_bits_horiz, |
| k, coeff); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() == 1) { |
| k = WarpHorizontalFilterLoopInterior<InnerCoeffUpdate, true, Filter>( |
| uint8x16_tag, int16x8_tag, base_ptr, horz_out, stride, stride2, stride3, |
| sx4, alpha, beta, p_height, i, round_const, reduce_bits_horiz, k, |
| coeff); |
| } |
| const auto src = hn::LoadU(uint8x16_tag, base_ptr + (k + 7) * stride); |
| if constexpr (InnerCoeffUpdate) { |
| int sx = sx4 + beta * (k + 4); |
| WarpHorizontalPrepareCoeffs<true, Filter, decltype(int16x8_tag)>( |
| alpha, beta, sx, coeff); |
| } |
| FilterPixelsHorizontal(uint8x16_tag, src, horz_out, coeff, round_const, |
| reduce_bits_horiz, k + 7); |
| } |
| |
| HWY_ATTR inline void UnpackWeightsAndSetRoundConst( |
| ConvolveParams *HWY_RESTRICT conv_params, const int round_bits, |
| const int offset_bits, IVec16 &HWY_RESTRICT res_sub_const, |
| IVec16 &HWY_RESTRICT round_bits_const, IVec16 &HWY_RESTRICT wt) { |
| res_sub_const = |
| hn::Set(int16xN_tag, -(1 << (offset_bits - conv_params->round_1)) - |
| (1 << (offset_bits - conv_params->round_1 - 1))); |
| round_bits_const = hn::Set(int16xN_tag, ((1 << round_bits) >> 1)); |
| |
| const auto w0 = static_cast<int16_t>(conv_params->fwd_offset); |
| const auto w1 = static_cast<int16_t>(conv_params->bck_offset); |
| const auto wt0 = hn::Set(int16xN_tag, w0); |
| const auto wt1 = hn::Set(int16xN_tag, w1); |
| wt = hn::InterleaveLower(wt0, wt1); |
| } |
| |
| HWY_ATTR HWY_INLINE IVec16 LoadAV1WarpedFilter(size_t offset) { |
| const WarpedFilterCoeff *HWY_RESTRICT warped_filter_ptr = |
| &av1_warped_filter[0][0]; |
| return hn::LoadN(int16xN_tag, |
| warped_filter_ptr + (offset >> WARPEDDIFF_PREC_BITS) * 8, 8); |
| } |
| |
| HWY_ATTR HWY_INLINE IVec16 LoadAV1WarpedFilterLower(size_t offset) { |
| const WarpedFilterCoeff *HWY_RESTRICT warped_filter_ptr = |
| &av1_warped_filter[0][0]; |
| return hn::ResizeBitCast( |
| int16xN_tag, |
| hn::Load(int16x8_tag, |
| warped_filter_ptr + (offset >> WARPEDDIFF_PREC_BITS) * 8)); |
| } |
| |
| template <int Block> |
| HWY_ATTR HWY_INLINE IVec16 LoadAV1WarpedFilterUpper(size_t offset, IVec16 src) { |
| const WarpedFilterCoeff *HWY_RESTRICT warped_filter_ptr = |
| &av1_warped_filter[0][0]; |
| return hn::InsertBlock<Block>( |
| src, hn::Load(int16x8_tag, |
| warped_filter_ptr + (offset >> WARPEDDIFF_PREC_BITS) * 8)); |
| } |
| |
| HWY_ATTR inline void PrepareVerticalFilterCoeffs(int gamma, int delta, int sy, |
| int16_t *HWY_RESTRICT coeffs) { |
| auto filt_00 = LoadAV1WarpedFilterLower(sy + 0 * gamma); |
| auto filt_01 = LoadAV1WarpedFilterLower(sy + 2 * gamma); |
| auto filt_02 = LoadAV1WarpedFilterLower(sy + 4 * gamma); |
| auto filt_03 = LoadAV1WarpedFilterLower(sy + 6 * gamma); |
| |
| if constexpr (int16xN_tag.MaxBlocks() >= 2) { |
| filt_00 = LoadAV1WarpedFilterUpper<1>(sy + delta + 0 * gamma, filt_00); |
| filt_01 = LoadAV1WarpedFilterUpper<1>(sy + delta + 2 * gamma, filt_01); |
| filt_02 = LoadAV1WarpedFilterUpper<1>(sy + delta + 4 * gamma, filt_02); |
| filt_03 = LoadAV1WarpedFilterUpper<1>(sy + delta + 6 * gamma, filt_03); |
| } |
| |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| filt_00 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 0 * gamma, filt_00); |
| filt_01 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 2 * gamma, filt_01); |
| filt_02 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 4 * gamma, filt_02); |
| filt_03 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 6 * gamma, filt_03); |
| |
| filt_00 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 0 * gamma, filt_00); |
| filt_01 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 2 * gamma, filt_01); |
| filt_02 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 4 * gamma, filt_02); |
| filt_03 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 6 * gamma, filt_03); |
| } |
| |
| auto filt_0 = hn::BitCast(int32xN_tag, filt_00); |
| auto filt_1 = hn::BitCast(int32xN_tag, filt_01); |
| auto filt_2 = hn::BitCast(int32xN_tag, filt_02); |
| auto filt_3 = hn::BitCast(int32xN_tag, filt_03); |
| |
| auto res_0 = hn::ZipLower(int64xN_tag, filt_0, filt_1); |
| auto res_1 = hn::ZipLower(int64xN_tag, filt_2, filt_3); |
| auto res_2 = hn::ZipUpper(int64xN_tag, filt_0, filt_1); |
| auto res_3 = hn::ZipUpper(int64xN_tag, filt_2, filt_3); |
| |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); |
| |
| filt_00 = LoadAV1WarpedFilterLower(sy + 1 * gamma); |
| filt_01 = LoadAV1WarpedFilterLower(sy + 3 * gamma); |
| filt_02 = LoadAV1WarpedFilterLower(sy + 5 * gamma); |
| filt_03 = LoadAV1WarpedFilterLower(sy + 7 * gamma); |
| |
| if constexpr (int16xN_tag.MaxBlocks() >= 2) { |
| filt_00 = LoadAV1WarpedFilterUpper<1>(sy + delta + 1 * gamma, filt_00); |
| filt_01 = LoadAV1WarpedFilterUpper<1>(sy + delta + 3 * gamma, filt_01); |
| filt_02 = LoadAV1WarpedFilterUpper<1>(sy + delta + 5 * gamma, filt_02); |
| filt_03 = LoadAV1WarpedFilterUpper<1>(sy + delta + 7 * gamma, filt_03); |
| } |
| |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| filt_00 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 1 * gamma, filt_00); |
| filt_01 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 3 * gamma, filt_01); |
| filt_02 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 5 * gamma, filt_02); |
| filt_03 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta + 7 * gamma, filt_03); |
| |
| filt_00 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 1 * gamma, filt_00); |
| filt_01 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 3 * gamma, filt_01); |
| filt_02 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 5 * gamma, filt_02); |
| filt_03 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta + 7 * gamma, filt_03); |
| } |
| |
| filt_0 = hn::BitCast(int32xN_tag, filt_00); |
| filt_1 = hn::BitCast(int32xN_tag, filt_01); |
| filt_2 = hn::BitCast(int32xN_tag, filt_02); |
| filt_3 = hn::BitCast(int32xN_tag, filt_03); |
| |
| res_0 = hn::ZipLower(int64xN_tag, filt_0, filt_1); |
| res_1 = hn::ZipLower(int64xN_tag, filt_2, filt_3); |
| res_2 = hn::ZipUpper(int64xN_tag, filt_0, filt_1); |
| res_3 = hn::ZipUpper(int64xN_tag, filt_2, filt_3); |
| |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); |
| } |
| |
| HWY_ATTR inline void PrepareVerticalFilterCoeffsDelta0( |
| int gamma, int delta, int sy, int16_t *HWY_RESTRICT coeffs) { |
| (void)delta; |
| auto filt_00 = LoadAV1WarpedFilter(sy + 0 * gamma); |
| auto filt_01 = LoadAV1WarpedFilter(sy + 2 * gamma); |
| auto filt_02 = LoadAV1WarpedFilter(sy + 4 * gamma); |
| auto filt_03 = LoadAV1WarpedFilter(sy + 6 * gamma); |
| |
| auto filt_10 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_00)); |
| auto filt_11 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_01)); |
| auto filt_12 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_02)); |
| auto filt_13 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_03)); |
| |
| auto res_0 = hn::ZipLower(int64xN_tag, filt_10, filt_11); |
| auto res_1 = hn::ZipLower(int64xN_tag, filt_12, filt_13); |
| auto res_2 = hn::ZipUpper(int64xN_tag, filt_10, filt_11); |
| auto res_3 = hn::ZipUpper(int64xN_tag, filt_12, filt_13); |
| |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); |
| |
| filt_00 = LoadAV1WarpedFilter(sy + 1 * gamma); |
| filt_01 = LoadAV1WarpedFilter(sy + 3 * gamma); |
| filt_02 = LoadAV1WarpedFilter(sy + 5 * gamma); |
| filt_03 = LoadAV1WarpedFilter(sy + 7 * gamma); |
| |
| filt_10 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_00)); |
| filt_11 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_01)); |
| filt_12 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_02)); |
| filt_13 = hn::BitCast(int32xN_tag, hn::BroadcastBlock<0>(filt_03)); |
| |
| res_0 = hn::ZipLower(int64xN_tag, filt_10, filt_11); |
| res_1 = hn::ZipLower(int64xN_tag, filt_12, filt_13); |
| res_2 = hn::ZipUpper(int64xN_tag, filt_10, filt_11); |
| res_3 = hn::ZipUpper(int64xN_tag, filt_12, filt_13); |
| |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_0, res_1)), |
| int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveLower(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); |
| hn::Store( |
| hn::BitCast(int16xN_tag, hn::InterleaveUpper(int64xN_tag, res_2, res_3)), |
| int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); |
| } |
| |
| HWY_ATTR inline void PrepareVerticalFilterCoeffsGamma0( |
| int gamma, int delta, int sy, int16_t *HWY_RESTRICT coeffs) { |
| (void)gamma; |
| auto filt_0 = LoadAV1WarpedFilterLower(sy); |
| if constexpr (int16xN_tag.MaxBlocks() >= 2) { |
| filt_0 = LoadAV1WarpedFilterUpper<1>(sy + delta, filt_0); |
| } |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| filt_0 = LoadAV1WarpedFilterUpper<2>(sy + 2 * delta, filt_0); |
| filt_0 = LoadAV1WarpedFilterUpper<3>(sy + 3 * delta, filt_0); |
| } |
| auto res_0 = hn::BitCast(int32xN_tag, filt_0); |
| |
| auto broadcast_0 = hn::BitCast(int16xN_tag, hn::Broadcast<0>(res_0)); |
| auto broadcast_1 = hn::BitCast(int16xN_tag, hn::Broadcast<1>(res_0)); |
| auto broadcast_2 = hn::BitCast(int16xN_tag, hn::Broadcast<2>(res_0)); |
| auto broadcast_3 = hn::BitCast(int16xN_tag, hn::Broadcast<3>(res_0)); |
| |
| hn::Store(broadcast_0, int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_1, int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_2, int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_3, int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_0, int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_1, int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_2, int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(broadcast_3, int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); |
| } |
| |
| HWY_ATTR inline void FilterPixelsVertical(int16_t *HWY_RESTRICT horz_out, |
| int16_t *HWY_RESTRICT src_lo, |
| int16_t *HWY_RESTRICT src_hi, |
| int16_t *HWY_RESTRICT coeffs, |
| IVec32 &HWY_RESTRICT res_lo, |
| IVec32 &HWY_RESTRICT res_hi, int row, |
| const IVec32 res_add_const) { |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| const auto horz_out_4 = |
| hn::Load(int16xN_tag, horz_out + (row + 4) * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_5 = hn::LoadU( |
| int16xN_tag, horz_out + (row + 5) * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_6 = hn::LoadU( |
| int16xN_tag, horz_out + (row + 6) * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_7 = hn::LoadU( |
| int16xN_tag, horz_out + (row + 7) * hn::MaxLanes(int16x8_tag)); |
| const auto src_lo_2 = |
| hn::InterleaveLower(int16xN_tag, horz_out_4, horz_out_5); |
| const auto src_hi_2 = |
| hn::InterleaveUpper(int16xN_tag, horz_out_4, horz_out_5); |
| const auto src_lo_3 = |
| hn::InterleaveLower(int16xN_tag, horz_out_6, horz_out_7); |
| const auto src_hi_3 = |
| hn::InterleaveUpper(int16xN_tag, horz_out_6, horz_out_7); |
| hn::Store(src_lo_2, int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_hi_2, int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_lo_3, int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_hi_3, int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); |
| } else if constexpr (int16xN_tag.MaxBlocks() == 2) { |
| const auto horz_out_6 = |
| hn::Load(int16xN_tag, horz_out + (row + 6) * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_7 = hn::LoadU( |
| int16xN_tag, horz_out + (row + 7) * hn::MaxLanes(int16x8_tag)); |
| const auto src_lo_3 = |
| hn::InterleaveLower(int16xN_tag, horz_out_6, horz_out_7); |
| const auto src_hi_3 = |
| hn::InterleaveUpper(int16xN_tag, horz_out_6, horz_out_7); |
| hn::Store(src_lo_3, int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_hi_3, int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); |
| } else if constexpr (int16xN_tag.MaxBlocks() == 1) { |
| const auto horz_out_6 = |
| hn::Load(int16x8_tag, horz_out + (row + 6) * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_7 = |
| hn::Load(int16x8_tag, horz_out + (row + 7) * hn::MaxLanes(int16x8_tag)); |
| const auto src_lo_3 = |
| hn::InterleaveLower(int16x8_tag, horz_out_6, horz_out_7); |
| const auto src_hi_3 = |
| hn::InterleaveUpper(int16x8_tag, horz_out_6, horz_out_7); |
| hn::Store(src_lo_3, int16x8_tag, src_lo + 3 * hn::MaxLanes(int16x8_tag)); |
| hn::Store(src_hi_3, int16x8_tag, src_hi + 3 * hn::MaxLanes(int16x8_tag)); |
| } |
| |
| const auto coeff_0 = |
| hn::Load(int16xN_tag, coeffs + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_1 = |
| hn::Load(int16xN_tag, coeffs + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_2 = |
| hn::Load(int16xN_tag, coeffs + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_3 = |
| hn::Load(int16xN_tag, coeffs + 3 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_4 = |
| hn::Load(int16xN_tag, coeffs + 4 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_5 = |
| hn::Load(int16xN_tag, coeffs + 5 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_6 = |
| hn::Load(int16xN_tag, coeffs + 6 * hn::MaxLanes(int16xN_tag)); |
| const auto coeff_7 = |
| hn::Load(int16xN_tag, coeffs + 7 * hn::MaxLanes(int16xN_tag)); |
| |
| const auto src_lo_0 = |
| hn::Load(int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_1 = |
| hn::Load(int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_2 = |
| hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_3 = |
| hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_0 = |
| hn::Load(int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_1 = |
| hn::Load(int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_2 = |
| hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_3 = |
| hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); |
| |
| auto even_sum0 = res_add_const; |
| auto even_sum1 = hn::Zero(int32xN_tag); |
| even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_0, coeff_0, |
| even_sum0, even_sum1); |
| even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_1, coeff_1, |
| even_sum0, even_sum1); |
| even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_2, coeff_2, |
| even_sum0, even_sum1); |
| even_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_lo_3, coeff_3, |
| even_sum0, even_sum1); |
| auto res_even = hn::RearrangeToOddPlusEven(even_sum0, even_sum1); |
| |
| auto odd_sum0 = res_add_const; |
| auto odd_sum1 = hn::Zero(int32xN_tag); |
| odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_0, coeff_4, |
| odd_sum0, odd_sum1); |
| odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_1, coeff_5, |
| odd_sum0, odd_sum1); |
| odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_2, coeff_6, |
| odd_sum0, odd_sum1); |
| odd_sum0 = hn::ReorderWidenMulAccumulate(int32xN_tag, src_hi_3, coeff_7, |
| odd_sum0, odd_sum1); |
| auto res_odd = hn::RearrangeToOddPlusEven(odd_sum0, odd_sum1); |
| |
| // Rearrange pixels back into the order 0 ... 7 |
| res_lo = hn::InterleaveLower(int32xN_tag, res_even, res_odd); |
| res_hi = hn::InterleaveUpper(int32xN_tag, res_even, res_odd); |
| } |
| |
| template <typename DS, typename DR, typename A, typename B, typename C> |
| HWY_ATTR HWY_INLINE void StoreRows(DS store_tag, DR row_tag, hn::VFromD<DR> vec, |
| A stride, B y, C x, |
| hn::TFromD<DS> *HWY_RESTRICT out) { |
| hn::TFromD<DS> *HWY_RESTRICT pointers[row_tag.MaxBlocks()]; |
| hn::TFromD<DS> *HWY_RESTRICT base_ptr = &out[y * stride + x]; |
| pointers[0] = base_ptr; |
| if constexpr (row_tag.MaxBlocks() >= 2) { |
| pointers[1] = base_ptr + stride; |
| } |
| if constexpr (row_tag.MaxBlocks() >= 3) { |
| pointers[2] = base_ptr + 2 * stride; |
| pointers[3] = base_ptr + 3 * stride; |
| } |
| hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<0>(vec)), store_tag, |
| pointers[0]); |
| if constexpr (row_tag.MaxBlocks() >= 2) { |
| hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<1>(vec)), store_tag, |
| pointers[1]); |
| } |
| if constexpr (row_tag.MaxBlocks() >= 3) { |
| hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<2>(vec)), store_tag, |
| pointers[2]); |
| hn::Store(hn::ResizeBitCast(store_tag, hn::ExtractBlock<3>(vec)), store_tag, |
| pointers[3]); |
| } |
| } |
| |
| HWY_ATTR HWY_INLINE void StoreVerticalFilterOutput( |
| IVec32 res_lo, IVec32 res_hi, const IVec16 wt, |
| const IVec16 res_sub_round_const, uint8_t *HWY_RESTRICT pred, |
| ConvolveParams *HWY_RESTRICT conv_params, int i, int j, int k, |
| const int reduce_bits_vert, int p_stride, int p_width, |
| const int round_bits) { |
| constexpr int kNumRows = uint16xN_tag.MaxBlocks(); |
| if (conv_params->is_compound) { |
| uint16_t *HWY_RESTRICT pointers[kNumRows]; |
| uint16_t *const base_ptr = |
| &conv_params->dst[(i + k) * conv_params->dst_stride + j]; |
| const int dst_stride = conv_params->dst_stride; |
| pointers[0] = base_ptr; |
| if constexpr (kNumRows >= 2) { |
| pointers[1] = base_ptr + dst_stride; |
| } |
| if constexpr (kNumRows >= 3) { |
| pointers[2] = base_ptr + 2 * dst_stride; |
| pointers[3] = base_ptr + 3 * dst_stride; |
| } |
| |
| res_lo = hn::ShiftRightSame(res_lo, reduce_bits_vert); |
| |
| if (p_width > 4) { |
| res_hi = hn::ShiftRightSame(res_hi, reduce_bits_vert); |
| const auto temp_16 = hn::ReorderDemote2To(uint16xN_tag, res_lo, res_hi); |
| if (conv_params->do_average) { |
| auto p_16 = hn::ResizeBitCast(uint16xN_tag, |
| hn::LoadU(uint16x8_tag, pointers[0])); |
| if constexpr (kNumRows >= 2) { |
| p_16 = hn::InsertBlock<1>( |
| p_16, hn::ResizeBitCast(uint16x8_tag, |
| hn::LoadU(uint16x8_tag, pointers[1]))); |
| } |
| if constexpr (kNumRows >= 3) { |
| p_16 = hn::InsertBlock<2>( |
| p_16, hn::ResizeBitCast(uint16x8_tag, |
| hn::LoadU(uint16x8_tag, pointers[2]))); |
| p_16 = hn::InsertBlock<3>( |
| p_16, hn::ResizeBitCast(uint16x8_tag, |
| hn::LoadU(uint16x8_tag, pointers[3]))); |
| } |
| auto res_16 = hn::Undefined(int16xN_tag); |
| if (conv_params->use_dist_wtd_comp_avg) { |
| const auto p_16_lo = hn::BitCast( |
| int16xN_tag, hn::InterleaveLower(uint16xN_tag, p_16, temp_16)); |
| const auto p_16_hi = hn::BitCast( |
| int16xN_tag, hn::InterleaveUpper(uint16xN_tag, p_16, temp_16)); |
| const auto wt_res_lo = |
| hn::WidenMulPairwiseAdd(int32xN_tag, p_16_lo, wt); |
| const auto wt_res_hi = |
| hn::WidenMulPairwiseAdd(int32xN_tag, p_16_hi, wt); |
| const auto shifted_lo_32 = |
| hn::ShiftRight<DIST_PRECISION_BITS>(wt_res_lo); |
| const auto shifted_hi_32 = |
| hn::ShiftRight<DIST_PRECISION_BITS>(wt_res_hi); |
| res_16 = hn::BitCast( |
| int16xN_tag, |
| hn::ReorderDemote2To(uint16xN_tag, shifted_lo_32, shifted_hi_32)); |
| } else { |
| res_16 = hn::ShiftRight<1>( |
| hn::BitCast(int16xN_tag, hn::Add(p_16, temp_16))); |
| } |
| res_16 = hn::ShiftRightSame(hn::Add(res_16, res_sub_round_const), |
| round_bits); |
| const auto res_8 = hn::ReorderDemote2To(uint8xN_tag, res_16, res_16); |
| StoreRows(uint8x8_tag, uint8xN_tag, res_8, p_stride, i + k, j, pred); |
| } else { |
| hn::StoreU( |
| hn::ResizeBitCast(uint16x8_tag, hn::ExtractBlock<0>(temp_16)), |
| uint16x8_tag, pointers[0]); |
| if constexpr (kNumRows >= 2) { |
| hn::StoreU( |
| hn::ResizeBitCast(uint16x8_tag, hn::ExtractBlock<1>(temp_16)), |
| uint16x8_tag, pointers[1]); |
| } |
| if constexpr (kNumRows >= 3) { |
| hn::StoreU( |
| hn::ResizeBitCast(uint16x8_tag, hn::ExtractBlock<2>(temp_16)), |
| uint16x8_tag, pointers[2]); |
| hn::StoreU( |
| hn::ResizeBitCast(uint16x8_tag, hn::ExtractBlock<3>(temp_16)), |
| uint16x8_tag, pointers[3]); |
| } |
| } |
| } else { |
| const auto temp_lo_16 = |
| hn::ReorderDemote2To(uint16xN_tag, res_lo, res_lo); |
| if (conv_params->do_average) { |
| auto p_16 = hn::ResizeBitCast(uint16xN_tag, |
| hn::LoadU(uint16x4_tag, pointers[0])); |
| if constexpr (kNumRows >= 2) { |
| p_16 = hn::InsertBlock<1>( |
| p_16, hn::ResizeBitCast(uint16x8_tag, |
| hn::LoadU(uint16x4_tag, pointers[1]))); |
| } |
| if constexpr (kNumRows >= 3) { |
| p_16 = hn::InsertBlock<2>( |
| p_16, hn::ResizeBitCast(uint16x8_tag, |
| hn::LoadU(uint16x4_tag, pointers[2]))); |
| p_16 = hn::InsertBlock<3>( |
| p_16, hn::ResizeBitCast(uint16x8_tag, |
| hn::LoadU(uint16x4_tag, pointers[3]))); |
| } |
| auto res_lo_16 = hn::Undefined(int16xN_tag); |
| if (conv_params->use_dist_wtd_comp_avg) { |
| const auto p_16_lo = hn::BitCast( |
| int16xN_tag, hn::InterleaveLower(uint16xN_tag, p_16, temp_lo_16)); |
| const auto wt_res_lo = |
| hn::WidenMulPairwiseAdd(int32xN_tag, p_16_lo, wt); |
| const auto shifted_32 = |
| hn::ShiftRight<DIST_PRECISION_BITS>(wt_res_lo); |
| res_lo_16 = hn::BitCast( |
| int16xN_tag, |
| hn::ReorderDemote2To(uint16xN_tag, shifted_32, shifted_32)); |
| } else { |
| res_lo_16 = hn::ShiftRight<1>( |
| hn::BitCast(int16xN_tag, hn::Add(p_16, temp_lo_16))); |
| } |
| res_lo_16 = hn::ShiftRightSame(hn::Add(res_lo_16, res_sub_round_const), |
| round_bits); |
| const auto res_8_lo = |
| hn::ReorderDemote2To(uint8xN_tag, res_lo_16, res_lo_16); |
| StoreRows(uint8x4_tag, uint8xN_tag, res_8_lo, p_stride, i + k, j, pred); |
| } else { |
| hn::Store( |
| hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<0>(temp_lo_16)), |
| uint16x4_tag, pointers[0]); |
| if constexpr (kNumRows >= 2) { |
| hn::Store( |
| hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<1>(temp_lo_16)), |
| uint16x4_tag, pointers[1]); |
| } |
| if constexpr (kNumRows >= 3) { |
| hn::Store( |
| hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<2>(temp_lo_16)), |
| uint16x4_tag, pointers[2]); |
| hn::Store( |
| hn::ResizeBitCast(uint16x4_tag, hn::ExtractBlock<3>(temp_lo_16)), |
| uint16x4_tag, pointers[3]); |
| } |
| } |
| } |
| } else { |
| const auto res_lo_round = hn::ShiftRightSame(res_lo, reduce_bits_vert); |
| const auto res_hi_round = hn::ShiftRightSame(res_hi, reduce_bits_vert); |
| |
| const auto res_16bit = |
| hn::ReorderDemote2To(int16xN_tag, res_lo_round, res_hi_round); |
| const auto res_8bit = |
| hn::ReorderDemote2To(uint8xN_tag, res_16bit, res_16bit); |
| // Store, blending with 'pred' if needed |
| if (p_width == 4) { |
| StoreRows(uint8x4_tag, uint8xN_tag, res_8bit, p_stride, i + k, j, pred); |
| } else { |
| StoreRows(uint8x8_tag, uint8xN_tag, res_8bit, p_stride, i + k, j, pred); |
| } |
| } |
| } |
| |
| enum class VerticalFilterCoeffs { kGamma0, kDelta0, kDefault }; |
| |
| template <VerticalFilterCoeffs Filter> |
| HWY_ATTR inline void WarpVerticalPrepareCoeffs(int gamma, int delta, int sy, |
| int16_t *HWY_RESTRICT coeffs) { |
| if constexpr (Filter == VerticalFilterCoeffs::kGamma0) { |
| PrepareVerticalFilterCoeffsGamma0(gamma, delta, sy, coeffs); |
| } else if constexpr (Filter == VerticalFilterCoeffs::kDelta0) { |
| PrepareVerticalFilterCoeffsDelta0(gamma, delta, sy, coeffs); |
| } else { |
| PrepareVerticalFilterCoeffs(gamma, delta, sy, coeffs); |
| } |
| } |
| |
| template <bool InnerCoeffUpdate, VerticalFilterCoeffs Filter> |
| HWY_ATTR inline void WarpVerticalFilterTemplate( |
| uint8_t *HWY_RESTRICT pred, int16_t *HWY_RESTRICT horz_out, |
| ConvolveParams *HWY_RESTRICT conv_params, int16_t gamma, int16_t delta, |
| int p_height, int p_stride, int p_width, int i, int j, int sy4, |
| const int reduce_bits_vert, const IVec32 res_add_const, |
| const int round_bits, const IVec16 res_sub_round_const, const IVec16 wt) { |
| HWY_ALIGN int16_t src_lo[4 * hn::MaxLanes(int16xN_tag)]; |
| HWY_ALIGN int16_t src_hi[4 * hn::MaxLanes(int16xN_tag)]; |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| const auto horz_out_0 = |
| hn::Load(int16xN_tag, horz_out + 0 * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_1 = |
| hn::LoadU(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_2 = |
| hn::LoadU(int16xN_tag, horz_out + 2 * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_3 = |
| hn::LoadU(int16xN_tag, horz_out + 3 * hn::MaxLanes(int16x8_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_0, horz_out_1), |
| int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_0, horz_out_1), |
| int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_2, horz_out_3), |
| int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_2, horz_out_3), |
| int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| } else if constexpr (int16xN_tag.MaxBlocks() == 2) { |
| const auto horz_out_0 = |
| hn::Load(int16xN_tag, horz_out + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_2 = |
| hn::Load(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_4 = |
| hn::Load(int16xN_tag, horz_out + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_1 = |
| hn::LoadU(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_3 = |
| hn::LoadU(int16xN_tag, horz_out + 3 * hn::MaxLanes(int16x8_tag)); |
| const auto horz_out_5 = |
| hn::LoadU(int16xN_tag, horz_out + 5 * hn::MaxLanes(int16x8_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_0, horz_out_1), |
| int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_0, horz_out_1), |
| int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_2, horz_out_3), |
| int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_2, horz_out_3), |
| int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_4, horz_out_5), |
| int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_4, horz_out_5), |
| int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| } else { |
| const auto horz_out_0 = |
| hn::Load(int16xN_tag, horz_out + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_1 = |
| hn::Load(int16xN_tag, horz_out + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_2 = |
| hn::Load(int16xN_tag, horz_out + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_3 = |
| hn::Load(int16xN_tag, horz_out + 3 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_4 = |
| hn::Load(int16xN_tag, horz_out + 4 * hn::MaxLanes(int16xN_tag)); |
| const auto horz_out_5 = |
| hn::Load(int16xN_tag, horz_out + 5 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_0, horz_out_1), |
| int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_0, horz_out_1), |
| int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_2, horz_out_3), |
| int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_2, horz_out_3), |
| int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveLower(int16xN_tag, horz_out_4, horz_out_5), |
| int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::InterleaveUpper(int16xN_tag, horz_out_4, horz_out_5), |
| int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| } |
| |
| HWY_ALIGN int16_t coeffs[8 * hn::MaxLanes(int16xN_tag)]; |
| if constexpr (!InnerCoeffUpdate) { |
| WarpVerticalPrepareCoeffs<Filter>(gamma, delta, sy4, coeffs); |
| } |
| |
| for (int k = -4; k < AOMMIN(4, p_height - i - 4); |
| k += static_cast<int>(int16xN_tag.MaxBlocks())) { |
| if constexpr (InnerCoeffUpdate) { |
| int sy = sy4 + delta * (k + 4); |
| WarpVerticalPrepareCoeffs<Filter>(gamma, delta, sy, coeffs); |
| } |
| |
| IVec32 res_lo, res_hi; |
| FilterPixelsVertical(horz_out, src_lo, src_hi, coeffs, res_lo, res_hi, |
| k + 4, res_add_const); |
| StoreVerticalFilterOutput(res_lo, res_hi, wt, res_sub_round_const, pred, |
| conv_params, i, j, k + 4, reduce_bits_vert, |
| p_stride, p_width, round_bits); |
| |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| hn::Store(hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| } else if constexpr (int16xN_tag.MaxBlocks() == 2) { |
| hn::Store(hn::Load(int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)), |
| int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| } else if constexpr (int16xN_tag.MaxBlocks() == 1) { |
| const auto src_lo_0 = |
| hn::Load(int16xN_tag, src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_1 = |
| hn::Load(int16xN_tag, src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_2 = |
| hn::Load(int16xN_tag, src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_3 = |
| hn::Load(int16xN_tag, src_lo + 3 * hn::MaxLanes(int16xN_tag)); |
| const auto src_lo_0_new = hn::InterleaveEven( |
| hn::ShiftRightLanes<1>(int16xN_tag, src_lo_0), src_lo_1); |
| const auto src_lo_1_new = hn::InterleaveEven( |
| hn::ShiftRightLanes<1>(int16xN_tag, src_lo_1), src_lo_2); |
| const auto src_lo_2_new = hn::InterleaveEven( |
| hn::ShiftRightLanes<1>(int16xN_tag, src_lo_2), src_lo_3); |
| hn::Store(src_lo_0_new, int16xN_tag, |
| src_lo + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_lo_1_new, int16xN_tag, |
| src_lo + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_lo_2_new, int16xN_tag, |
| src_lo + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_0 = |
| hn::Load(int16xN_tag, src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_1 = |
| hn::Load(int16xN_tag, src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_2 = |
| hn::Load(int16xN_tag, src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_3 = |
| hn::Load(int16xN_tag, src_hi + 3 * hn::MaxLanes(int16xN_tag)); |
| const auto src_hi_0_new = hn::InterleaveEven( |
| hn::ShiftRightLanes<1>(int16xN_tag, src_hi_0), src_hi_1); |
| const auto src_hi_1_new = hn::InterleaveEven( |
| hn::ShiftRightLanes<1>(int16xN_tag, src_hi_1), src_hi_2); |
| const auto src_hi_2_new = hn::InterleaveEven( |
| hn::ShiftRightLanes<1>(int16xN_tag, src_hi_2), src_hi_3); |
| hn::Store(src_hi_0_new, int16xN_tag, |
| src_hi + 0 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_hi_1_new, int16xN_tag, |
| src_hi + 1 * hn::MaxLanes(int16xN_tag)); |
| hn::Store(src_hi_2_new, int16xN_tag, |
| src_hi + 2 * hn::MaxLanes(int16xN_tag)); |
| } |
| } |
| } |
| |
| HWY_ATTR inline void PrepareWarpVerticalFilter( |
| uint8_t *HWY_RESTRICT pred, int16_t *HWY_RESTRICT horz_out, |
| ConvolveParams *HWY_RESTRICT conv_params, int16_t gamma, int16_t delta, |
| int p_height, int p_stride, int p_width, int i, int j, int sy4, |
| const int reduce_bits_vert, const IVec32 res_add_const, |
| const int round_bits, const IVec16 res_sub_round_const, const IVec16 wt) { |
| if (gamma == 0 && delta == 0) |
| WarpVerticalFilterTemplate<false, VerticalFilterCoeffs::kGamma0>( |
| pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, |
| i, j, sy4, reduce_bits_vert, res_add_const, round_bits, |
| res_sub_round_const, wt); |
| else if (gamma == 0 && delta != 0) |
| WarpVerticalFilterTemplate<true, VerticalFilterCoeffs::kGamma0>( |
| pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, |
| i, j, sy4, reduce_bits_vert, res_add_const, round_bits, |
| res_sub_round_const, wt); |
| else if (gamma != 0 && delta == 0) |
| WarpVerticalFilterTemplate<false, VerticalFilterCoeffs::kDelta0>( |
| pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, |
| i, j, sy4, reduce_bits_vert, res_add_const, round_bits, |
| res_sub_round_const, wt); |
| else |
| WarpVerticalFilterTemplate<true, VerticalFilterCoeffs::kDefault>( |
| pred, horz_out, conv_params, gamma, delta, p_height, p_stride, p_width, |
| i, j, sy4, reduce_bits_vert, res_add_const, round_bits, |
| res_sub_round_const, wt); |
| } |
| |
| HWY_ATTR inline void PrepareWarpHorizontalFilter( |
| const uint8_t *HWY_RESTRICT ref, int16_t *HWY_RESTRICT horz_out, int stride, |
| int32_t ix4, int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, |
| int height, int i, const IVec16 round_const, const int reduce_bits_horiz) { |
| if (alpha == 0 && beta == 0) |
| WarpHorizontalFilterTemplate<false, HorizontalFilterCoeffs::kAlpha0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, |
| round_const, reduce_bits_horiz); |
| else if (alpha == 0 && beta != 0) |
| WarpHorizontalFilterTemplate<true, HorizontalFilterCoeffs::kAlpha0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, |
| round_const, reduce_bits_horiz); |
| else if (alpha != 0 && beta == 0) |
| WarpHorizontalFilterTemplate<false, HorizontalFilterCoeffs::kBeta0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, |
| round_const, reduce_bits_horiz); |
| else |
| WarpHorizontalFilterTemplate<true, HorizontalFilterCoeffs::kDefault>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, height, i, |
| round_const, reduce_bits_horiz); |
| } |
| |
| template <typename D> |
| HWY_ATTR HWY_INLINE int WarpHorizontalFilterOutOfBoundsSetLoop( |
| D tag, const uint8_t *HWY_RESTRICT ref, int height, int stride, |
| int p_height, int i, int iy4, int16_t const4, int16_t const5, int offset, |
| int k, int16_t *HWY_RESTRICT horz_out) { |
| constexpr int kNumRows = tag.MaxBlocks(); |
| for (; k < AOMMIN(8, p_height - i) - kNumRows; k += kNumRows) { |
| int iy = clamp(iy4 + k + 0, 0, height - 1); |
| auto src = hn::ResizeBitCast( |
| tag, hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); |
| if constexpr (kNumRows >= 2) { |
| iy = clamp(iy4 + k + 1, 0, height - 1); |
| src = hn::InsertBlock<1>( |
| src, |
| hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); |
| } |
| if constexpr (kNumRows >= 3) { |
| iy = clamp(iy4 + k + 2, 0, height - 1); |
| src = hn::InsertBlock<2>( |
| src, |
| hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); |
| iy = clamp(iy4 + k + 3, 0, height - 1); |
| src = hn::InsertBlock<3>( |
| src, |
| hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5)); |
| } |
| hn::Store(src, tag, horz_out + (k + 7) * hn::MaxLanes(int16x8_tag)); |
| } |
| return k; |
| } |
| |
| HWY_ATTR void WarpHorizontalFilterOutOfBoundsSet( |
| const uint8_t *HWY_RESTRICT ref, int height, int stride, int p_height, |
| int i, int iy4, int16_t const4, int16_t const5, int offset, |
| int16_t *HWY_RESTRICT horz_out) { |
| int k = -7, iy; |
| if constexpr (int16xN_tag.MaxBlocks() >= 3) { |
| k = WarpHorizontalFilterOutOfBoundsSetLoop(int16xN_tag, ref, height, stride, |
| p_height, i, iy4, const4, const5, |
| offset, k, horz_out); |
| } |
| if constexpr (int16xN_tag.MaxBlocks() >= 2) { |
| k = WarpHorizontalFilterOutOfBoundsSetLoop(int16x16_tag, ref, height, |
| stride, p_height, i, iy4, const4, |
| const5, offset, k, horz_out); |
| } |
| if constexpr (int16xN_tag.MaxBlocks() == 1) { |
| k = WarpHorizontalFilterOutOfBoundsSetLoop(int16x8_tag, ref, height, stride, |
| p_height, i, iy4, const4, const5, |
| offset, k, horz_out); |
| } |
| iy = iy4 + k; |
| iy = clamp(iy4 + k, 0, height - 1); |
| hn::Store(hn::Set(int16x8_tag, const4 + ref[iy * stride + offset] * const5), |
| int16x8_tag, horz_out + (k + 7) * hn::MaxLanes(int16x8_tag)); |
| } |
| |
| template <typename D> |
| HWY_ATTR int WarpHorizontalFilterOutOfBoundsPadLoop( |
| D tag, const uint8_t *HWY_RESTRICT ref, int stride, int32_t ix4, |
| int32_t iy4, int32_t sx4, int alpha, int beta, int p_height, int height, |
| int i, const IVec16 round_const, const int reduce_bits_horiz, |
| int out_of_boundary_left, int out_of_boundary_right, int k, |
| int16_t *HWY_RESTRICT horz_out) { |
| constexpr int kNumRows = tag.MaxBlocks(); |
| for (; k < (AOMMIN(8, p_height - i) - kNumRows); k += kNumRows) { |
| auto src = LoadRowsClamped(tag, ref + ix4 - 7, stride, iy4 + k, height); |
| if (out_of_boundary_left >= 0) { |
| const auto shuffle_reg_left = |
| hn::LoadDup128(tag, warp_pad_left[out_of_boundary_left]); |
| src = hn::TableLookupBytes(src, shuffle_reg_left); |
| } |
| if (out_of_boundary_right >= 0) { |
| const auto shuffle_reg_right = |
| hn::LoadDup128(tag, warp_pad_right[out_of_boundary_right]); |
| src = hn::TableLookupBytes(src, shuffle_reg_right); |
| } |
| int sx = sx4 + beta * (k + 4); |
| HorizontalFilter(tag, src, horz_out, sx, alpha, beta, k + 7, round_const, |
| reduce_bits_horiz); |
| } |
| return k; |
| } |
| |
| HWY_ATTR void WarpHorizontalFilterOutOfBoundsPad( |
| const uint8_t *HWY_RESTRICT ref, int stride, int32_t ix4, int32_t iy4, |
| int32_t sx4, int alpha, int beta, int p_height, int width, int height, |
| int i, const IVec16 round_const, const int reduce_bits_horiz, |
| int16_t *HWY_RESTRICT horz_out) { |
| const int out_of_boundary_left = -(ix4 - 6); |
| const int out_of_boundary_right = (ix4 + 8) - width; |
| int k = -7, iy, sx; |
| if constexpr (uint8xN_tag.MaxBlocks() >= 3) { |
| k = WarpHorizontalFilterOutOfBoundsPadLoop( |
| uint8xN_tag, ref, stride, ix4, iy4, sx4, alpha, beta, p_height, height, |
| i, round_const, reduce_bits_horiz, out_of_boundary_left, |
| out_of_boundary_right, k, horz_out); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() >= 2) { |
| k = WarpHorizontalFilterOutOfBoundsPadLoop( |
| uint8x32_tag, ref, stride, ix4, iy4, sx4, alpha, beta, p_height, height, |
| i, round_const, reduce_bits_horiz, out_of_boundary_left, |
| out_of_boundary_right, k, horz_out); |
| } |
| if constexpr (uint8xN_tag.MaxBlocks() == 1) { |
| k = WarpHorizontalFilterOutOfBoundsPadLoop( |
| uint8xN_tag, ref, stride, ix4, iy4, sx4, alpha, beta, p_height, height, |
| i, round_const, reduce_bits_horiz, out_of_boundary_left, |
| out_of_boundary_right, k, horz_out); |
| } |
| iy = iy4 + k; |
| iy = clamp(iy, 0, height - 1); |
| auto src = hn::LoadU(uint8x16_tag, ref + iy * stride + ix4 - 7); |
| if (out_of_boundary_left >= 0) { |
| const auto shuffle_reg_left = |
| hn::LoadU(uint8x16_tag, warp_pad_left[out_of_boundary_left]); |
| src = hn::TableLookupBytes(src, shuffle_reg_left); |
| } |
| if (out_of_boundary_right >= 0) { |
| const auto shuffle_reg_right = |
| hn::LoadU(uint8x16_tag, warp_pad_right[out_of_boundary_right]); |
| src = hn::TableLookupBytes(src, shuffle_reg_right); |
| } |
| sx = sx4 + beta * (k + 4); |
| HWY_ALIGN int8_t coeff[4 * hn::MaxLanes(coeff_tag)]; |
| PrepareLastHorizontalFilterCoefficients(int16xN_tag, alpha, beta, sx, coeff); |
| FilterPixelsHorizontal(uint8x16_tag, src, horz_out, coeff, round_const, |
| reduce_bits_horiz, k + 7); |
| } |
| |
| template <bool HorizInnerCoeffUpdate, HorizontalFilterCoeffs HorizFilter, |
| bool VertInnerCoeffUpdate, VerticalFilterCoeffs VertFilter> |
| HWY_ATTR inline void WarpAffineTemplate( |
| const int32_t *HWY_RESTRICT mat, const uint8_t *HWY_RESTRICT ref, int width, |
| int height, int stride, uint8_t *HWY_RESTRICT pred, int p_col, int p_row, |
| int p_width, int p_height, int p_stride, int subsampling_x, |
| int subsampling_y, ConvolveParams *HWY_RESTRICT conv_params, int16_t alpha, |
| int16_t beta, int16_t gamma, int16_t delta) { |
| int i, j; |
| const int bd = 8; |
| const int reduce_bits_horiz = conv_params->round_0; |
| const int reduce_bits_vert = conv_params->is_compound |
| ? conv_params->round_1 |
| : 2 * FILTER_BITS - reduce_bits_horiz; |
| const int offset_bits_horiz = bd + FILTER_BITS - 1; |
| assert(IMPLIES(conv_params->is_compound, conv_params->dst != NULL)); |
| |
| const int offset_bits_vert = bd + 2 * FILTER_BITS - reduce_bits_horiz; |
| const auto reduce_bits_vert_const = |
| hn::Set(int32xN_tag, ((1 << reduce_bits_vert) >> 1)); |
| const auto res_add_const = hn::Set(int32xN_tag, 1 << offset_bits_vert); |
| const int round_bits = |
| 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
| const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
| assert(IMPLIES(conv_params->do_average, conv_params->is_compound)); |
| |
| const auto round_const = hn::Set( |
| int16xN_tag, (1 << offset_bits_horiz) + ((1 << reduce_bits_horiz) >> 1)); |
| |
| IVec16 res_sub_const, round_bits_const, wt; |
| UnpackWeightsAndSetRoundConst(conv_params, round_bits, offset_bits, |
| res_sub_const, round_bits_const, wt); |
| const auto res_sub_round_const = hn::Add(res_sub_const, round_bits_const); |
| |
| IVec32 res_add_const_1; |
| if (conv_params->is_compound == 1) { |
| res_add_const_1 = hn::Add(reduce_bits_vert_const, res_add_const); |
| } else { |
| res_add_const_1 = hn::Set(int32xN_tag, -(1 << (bd + reduce_bits_vert - 1)) + |
| ((1 << reduce_bits_vert) >> 1)); |
| } |
| const int32_t const1 = alpha * (-4) + beta * (-4) + |
| (1 << (WARPEDDIFF_PREC_BITS - 1)) + |
| (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); |
| const int32_t const2 = gamma * (-4) + delta * (-4) + |
| (1 << (WARPEDDIFF_PREC_BITS - 1)) + |
| (WARPEDPIXEL_PREC_SHIFTS << WARPEDDIFF_PREC_BITS); |
| const int32_t const3 = ((1 << WARP_PARAM_REDUCE_BITS) - 1); |
| const int16_t const4 = (1 << (bd + FILTER_BITS - reduce_bits_horiz - 1)); |
| const int16_t const5 = (1 << (FILTER_BITS - reduce_bits_horiz)); |
| |
| const int64_t dst_x_stride_j = |
| static_cast<int64_t>(mat[2]) * (8 << subsampling_x); |
| const int64_t dst_y_stride_j = |
| static_cast<int64_t>(mat[4]) * (8 << subsampling_x); |
| const int64_t dst_x_stride_i = |
| static_cast<int64_t>(mat[3]) * (8 << subsampling_y); |
| const int64_t dst_y_stride_i = |
| static_cast<int64_t>(mat[5]) * (8 << subsampling_y); |
| |
| const int32_t src_x_0 = (p_col + 4) << subsampling_x; |
| const int32_t src_y_0 = (p_row + 4) << subsampling_y; |
| const int64_t dst_x_start = static_cast<int64_t>(mat[2]) * src_x_0 + |
| static_cast<int64_t>(mat[3]) * src_y_0 + |
| static_cast<int64_t>(mat[0]); |
| const int64_t dst_y_start = static_cast<int64_t>(mat[4]) * src_x_0 + |
| static_cast<int64_t>(mat[5]) * src_y_0 + |
| static_cast<int64_t>(mat[1]); |
| |
| int64_t dst_x_row = dst_x_start; |
| int64_t dst_y_row = dst_y_start; |
| |
| for (i = 0; i < p_height; i += 8) { |
| int64_t dst_x = dst_x_row; |
| int64_t dst_y = dst_y_row; |
| for (j = 0; j < p_width; j += 8) { |
| HWY_ALIGN int16_t horz_out[8 * 16 + hn::MaxLanes(int16xN_tag)]; |
| const int64_t x4 = dst_x >> subsampling_x; |
| const int64_t y4 = dst_y >> subsampling_y; |
| |
| int32_t ix4 = static_cast<int32_t>(x4 >> WARPEDMODEL_PREC_BITS); |
| int32_t sx4 = x4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); |
| int32_t iy4 = static_cast<int32_t>(y4 >> WARPEDMODEL_PREC_BITS); |
| int32_t sy4 = y4 & ((1 << WARPEDMODEL_PREC_BITS) - 1); |
| |
| // Add in all the constant terms, including rounding and offset |
| sx4 += const1; |
| sy4 += const2; |
| |
| sx4 &= ~const3; |
| sy4 &= ~const3; |
| |
| // Horizontal filter |
| // If the block is aligned such that, after clamping, every sample |
| // would be taken from the leftmost/rightmost column, then we can |
| // skip the expensive horizontal filter. |
| |
| if (ix4 <= -7) { |
| WarpHorizontalFilterOutOfBoundsSet(ref, height, stride, p_height, i, |
| iy4, const4, const5, 0, horz_out); |
| } else if (ix4 >= width + 6) { |
| WarpHorizontalFilterOutOfBoundsSet(ref, height, stride, p_height, i, |
| iy4, const4, const5, width - 1, |
| horz_out); |
| } else if (((ix4 - 7) < 0) || ((ix4 + 9) > width)) { |
| WarpHorizontalFilterOutOfBoundsPad( |
| ref, stride, ix4, iy4, sx4, alpha, beta, p_height, width, height, i, |
| round_const, reduce_bits_horiz, horz_out); |
| } else { |
| if (iy4 - 7 >= 0 && iy4 + 7 < height) { |
| if constexpr (HorizFilter == HorizontalFilterCoeffs::kAlpha0) { |
| WarpHorizontalFilterTemplateInterior< |
| HorizInnerCoeffUpdate, HorizontalFilterCoeffs::kAlpha0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, i, |
| round_const, reduce_bits_horiz); |
| } else if constexpr (HorizFilter == HorizontalFilterCoeffs::kBeta0) { |
| WarpHorizontalFilterTemplateInterior< |
| HorizInnerCoeffUpdate, HorizontalFilterCoeffs::kBeta0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, i, |
| round_const, reduce_bits_horiz); |
| } else { |
| WarpHorizontalFilterTemplateInterior< |
| HorizInnerCoeffUpdate, HorizontalFilterCoeffs::kDefault>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, i, |
| round_const, reduce_bits_horiz); |
| } |
| } else { |
| if constexpr (HorizFilter == HorizontalFilterCoeffs::kAlpha0) { |
| WarpHorizontalFilterTemplate<HorizInnerCoeffUpdate, |
| HorizontalFilterCoeffs::kAlpha0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, |
| height, i, round_const, reduce_bits_horiz); |
| } else if constexpr (HorizFilter == HorizontalFilterCoeffs::kBeta0) { |
| WarpHorizontalFilterTemplate<HorizInnerCoeffUpdate, |
| HorizontalFilterCoeffs::kBeta0>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, |
| height, i, round_const, reduce_bits_horiz); |
| } else { |
| WarpHorizontalFilterTemplate<HorizInnerCoeffUpdate, |
| HorizontalFilterCoeffs::kDefault>( |
| ref, horz_out, stride, ix4, iy4, sx4, alpha, beta, p_height, |
| height, i, round_const, reduce_bits_horiz); |
| } |
| } |
| } |
| |
| // Vertical filter |
| WarpVerticalFilterTemplate<VertInnerCoeffUpdate, VertFilter>( |
| pred, horz_out, conv_params, gamma, delta, p_height, p_stride, |
| p_width, i, j, sy4, reduce_bits_vert, res_add_const_1, round_bits, |
| res_sub_round_const, wt); |
| |
| dst_x += dst_x_stride_j; |
| dst_y += dst_y_stride_j; |
| } |
| dst_x_row += dst_x_stride_i; |
| dst_y_row += dst_y_stride_i; |
| } |
| } |
| |
| template <bool HorizInnerCoeffUpdate, HorizontalFilterCoeffs HorizFilter> |
| HWY_ATTR inline void WarpAffineHorizDispatched( |
| const int32_t *HWY_RESTRICT mat, const uint8_t *HWY_RESTRICT ref, int width, |
| int height, int stride, uint8_t *HWY_RESTRICT pred, int p_col, int p_row, |
| int p_width, int p_height, int p_stride, int subsampling_x, |
| int subsampling_y, ConvolveParams *HWY_RESTRICT conv_params, int16_t alpha, |
| int16_t beta, int16_t gamma, int16_t delta) { |
| if (gamma == 0 && delta == 0) { |
| WarpAffineTemplate<HorizInnerCoeffUpdate, HorizFilter, false, |
| VerticalFilterCoeffs::kGamma0>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } else if (gamma == 0 && delta != 0) { |
| WarpAffineTemplate<HorizInnerCoeffUpdate, HorizFilter, true, |
| VerticalFilterCoeffs::kGamma0>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } else if (gamma != 0 && delta == 0) { |
| WarpAffineTemplate<HorizInnerCoeffUpdate, HorizFilter, false, |
| VerticalFilterCoeffs::kDelta0>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } else { |
| WarpAffineTemplate<HorizInnerCoeffUpdate, HorizFilter, true, |
| VerticalFilterCoeffs::kDefault>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } |
| } |
| |
| HWY_ATTR void WarpAffine(const int32_t *HWY_RESTRICT mat, |
| const uint8_t *HWY_RESTRICT ref, int width, int height, |
| int stride, uint8_t *HWY_RESTRICT pred, int p_col, |
| int p_row, int p_width, int p_height, int p_stride, |
| int subsampling_x, int subsampling_y, |
| ConvolveParams *HWY_RESTRICT conv_params, |
| int16_t alpha, int16_t beta, int16_t gamma, |
| int16_t delta) { |
| if (alpha == 0 && beta == 0) { |
| WarpAffineHorizDispatched<false, HorizontalFilterCoeffs::kAlpha0>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } else if (alpha == 0 && beta != 0) { |
| WarpAffineHorizDispatched<true, HorizontalFilterCoeffs::kAlpha0>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } else if (alpha != 0 && beta == 0) { |
| WarpAffineHorizDispatched<false, HorizontalFilterCoeffs::kBeta0>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } else { |
| WarpAffineHorizDispatched<true, HorizontalFilterCoeffs::kDefault>( |
| mat, ref, width, height, stride, pred, p_col, p_row, p_width, p_height, |
| p_stride, subsampling_x, subsampling_y, conv_params, alpha, beta, gamma, |
| delta); |
| } |
| } |
| |
| } // namespace HWY_NAMESPACE |
| } // namespace |
| |
| #define MAKE_WARP_AFFINE(suffix) \ |
| extern "C" void av1_warp_affine_##suffix( \ |
| const int32_t *HWY_RESTRICT mat, const uint8_t *HWY_RESTRICT ref, \ |
| int width, int height, int stride, uint8_t *HWY_RESTRICT pred, \ |
| int p_col, int p_row, int p_width, int p_height, int p_stride, \ |
| int subsampling_x, int subsampling_y, \ |
| ConvolveParams *HWY_RESTRICT conv_params, int16_t alpha, int16_t beta, \ |
| int16_t gamma, int16_t delta); \ |
| HWY_ATTR void av1_warp_affine_##suffix( \ |
| const int32_t *HWY_RESTRICT mat, const uint8_t *HWY_RESTRICT ref, \ |
| int width, int height, int stride, uint8_t *HWY_RESTRICT pred, \ |
| int p_col, int p_row, int p_width, int p_height, int p_stride, \ |
| int subsampling_x, int subsampling_y, \ |
| ConvolveParams *HWY_RESTRICT conv_params, int16_t alpha, int16_t beta, \ |
| int16_t gamma, int16_t delta) { \ |
| HWY_NAMESPACE::WarpAffine(mat, ref, width, height, stride, pred, p_col, \ |
| p_row, p_width, p_height, p_stride, \ |
| subsampling_x, subsampling_y, conv_params, \ |
| alpha, beta, gamma, delta); \ |
| } |
| |
| HWY_AFTER_NAMESPACE(); |
| |
| #endif // AOM_AV1_COMMON_WARP_PLANE_HWY_H_ |
