Google Git Google Git
Sign in
aomedia/aom/fb718313902eaf2be17e86cc158374dfd3fc2a06/./aom_dsp/convolve_hwy.h
blob: a93c65b186aa8d74853ed148a6709228c3d83327 [file]
/*
* Copyright (c) 2026, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_AOM_DSP_CONVOLVE_HWY_H_
#define AOM_AOM_DSP_CONVOLVE_HWY_H_
#include "config/aom_config.h"
#include <cassert>
#include "aom_dsp/arm/aom_filter.h"
#include "third_party/highway/hwy/highway.h"
#if HAVE_SSSE3
extern "C" void aom_convolve8_horiz_ssse3(
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4, int w, int h);
#endif
HWY_BEFORE_NAMESPACE();
namespace {
namespace HWY_NAMESPACE {
namespace hn = hwy::HWY_NAMESPACE;
template <typename T16_8, typename T8_16, typename V8_16, typename M,
typename VC>
HWY_ATTR HWY_INLINE hn::VFromD<T16_8> ComputeHoriz8TapSum(
T16_8 tag16_8, T8_16 tag8_16, const V8_16 &d0, const V8_16 &d2,
const V8_16 &d4, const V8_16 &d6, const M &shuffle_mask, const VC &coeff01,
const VC &coeff23, const VC &coeff45, const VC &coeff67) {
(void)tag8_16;
return hn::Add(
hn::Add(hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d0, shuffle_mask), coeff01),
hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d2, shuffle_mask), coeff23)),
hn::Add(hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d4, shuffle_mask), coeff45),
hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d6, shuffle_mask), coeff67)));
}
template <typename T16_8, typename T8_16, typename V8_16, typename M,
typename VC>
HWY_ATTR HWY_INLINE hn::VFromD<T16_8> ComputeHoriz4TapSum(
T16_8 tag16_8, T8_16 tag8_16, const V8_16 &d0, const V8_16 &d2,
const M &shuffle_mask, const VC &coeff23, const VC &coeff45) {
(void)tag8_16;
return hn::Add(hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d0, shuffle_mask), coeff23),
hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d2, shuffle_mask), coeff45));
}
template <int chunk_size, typename T8_8, typename T8_4, typename V16_8>
HWY_ATTR HWY_INLINE void StoreOutputChunk(T8_8 tag8_8, T8_4 tag8_4,
const V16_8 &sum,
const V16_8 &bias_val, uint8_t *dst) {
auto res = hn::ShiftRight<FILTER_BITS - 1>(hn::Add(sum, bias_val));
if (chunk_size == 4) {
hn::StoreU(hn::LowerHalf(tag8_4, hn::DemoteTo(tag8_8, res)), tag8_4, dst);
} else {
// chunk_size == 8
hn::StoreU(hn::DemoteTo(tag8_8, res), tag8_8, dst);
}
}
template <typename T16_8, typename T8_16, typename M, typename VC>
HWY_ATTR HWY_INLINE hn::VFromD<T16_8> ComputeChunkSum2Tap(
T16_8 tag16_8, T8_16 tag8_16, const uint8_t *src, int j,
const M &shuffle_mask, const VC &coeff) {
(void)tag8_16;
auto d0 = hn::LoadU(tag8_16, src + j);
return hn::SatWidenMulPairwiseAdd(
tag16_8, hn::TableLookupBytes(d0, shuffle_mask), coeff);
}
template <typename T16_8, typename T8_16, typename M, typename VC>
HWY_ATTR HWY_INLINE hn::VFromD<T16_8> ComputeChunkSum4Tap(
T16_8 tag16_8, T8_16 tag8_16, const uint8_t *src, int j,
const M &shuffle_mask, const VC &coeff0, const VC &coeff1) {
auto d0 = hn::LoadU(tag8_16, src + j + 0);
auto d2 = hn::LoadU(tag8_16, src + j + 2);
return ComputeHoriz4TapSum(tag16_8, tag8_16, d0, d2, shuffle_mask, coeff0,
coeff1);
}
template <typename T16_8, typename T8_16, typename M, typename VC>
HWY_ATTR HWY_INLINE hn::VFromD<T16_8> ComputeChunkSum8Tap(
T16_8 tag16_8, T8_16 tag8_16, const uint8_t *src, int j,
const M &shuffle_mask, const VC &coeff0, const VC &coeff1, const VC &coeff2,
const VC &coeff3) {
auto d0 = hn::LoadU(tag8_16, src + j + 0);
auto d2 = hn::LoadU(tag8_16, src + j + 2);
auto d4 = hn::LoadU(tag8_16, src + j + 4);
auto d6 = hn::LoadU(tag8_16, src + j + 6);
return ComputeHoriz8TapSum(tag16_8, tag8_16, d0, d2, d4, d6, shuffle_mask,
coeff0, coeff1, coeff2, coeff3);
}
template <int chunk_size, typename T8_8, typename T8_4, typename V16_8,
typename SumComputer>
HWY_ATTR HWY_INLINE void Process2RowsChunk(T8_8 tag8_8, T8_4 tag8_4,
const uint8_t *src,
ptrdiff_t src_stride, uint8_t *dst,
ptrdiff_t dst_stride, int j,
const V16_8 &bias_val,
SumComputer sum_computer) {
auto r0_sum = sum_computer(src, j);
auto r1_sum = sum_computer(src + src_stride, j);
StoreOutputChunk<chunk_size>(tag8_8, tag8_4, r0_sum, bias_val, dst + j);
StoreOutputChunk<chunk_size>(tag8_8, tag8_4, r1_sum, bias_val,
dst + dst_stride + j);
}
template <int chunk_size, typename T8_8, typename T8_4, typename V16_8,
typename SumComputer>
HWY_ATTR HWY_INLINE void Process1RowChunk(T8_8 tag8_8, T8_4 tag8_4,
const uint8_t *src, uint8_t *dst,
int j, const V16_8 &bias_val,
SumComputer sum_computer) {
auto r0_sum = sum_computer(src, j);
StoreOutputChunk<chunk_size>(tag8_8, tag8_4, r0_sum, bias_val, dst + j);
}
template <typename D>
HWY_ATTR HWY_INLINE hn::VFromD<D> LoadUnaligned4x4(D tag16, const uint8_t *buf,
ptrdiff_t stride) {
hn::CappedTag<uint32_t, 4> tag32;
uint32_t r0, r1, r2, r3;
memcpy(&r0, buf, 4);
memcpy(&r1, buf + stride, 4);
memcpy(&r2, buf + 2 * stride, 4);
memcpy(&r3, buf + 3 * stride, 4);
auto v32 = hn::Zero(tag32);
v32 = hn::InsertLane(v32, 0, r0);
v32 = hn::InsertLane(v32, 1, r1);
v32 = hn::InsertLane(v32, 2, r2);
v32 = hn::InsertLane(v32, 3, r3);
hn::Rebind<uint8_t, D> tag8;
return hn::PromoteTo(tag16, hn::BitCast(tag8, v32));
}
template <typename D>
HWY_ATTR HWY_INLINE void StoreUnaligned4x4(D tag16, uint8_t *buf,
ptrdiff_t stride,
hn::VFromD<D> &vec) {
(void)tag16;
hn::Rebind<uint8_t, D> tag8;
constexpr hn::Half<D> half_tag16;
auto vec_demoted = hn::ReorderDemote2To(tag8, hn::LowerHalf(half_tag16, vec),
hn::UpperHalf(half_tag16, vec));
constexpr hn::Half<decltype(tag8)> half_tag;
constexpr hn::Half<decltype(half_tag)> quarter_tag;
auto vec1_2 = hn::LowerHalf(half_tag, vec_demoted);
auto vec2_2 = hn::UpperHalf(half_tag, vec_demoted);
auto vec1_4 = hn::LowerHalf(quarter_tag, vec1_2);
auto vec2_4 = hn::UpperHalf(quarter_tag, vec1_2);
auto vec3_4 = hn::LowerHalf(quarter_tag, vec2_2);
auto vec4_4 = hn::UpperHalf(quarter_tag, vec2_2);
hn::StoreU(vec1_4, quarter_tag, buf);
hn::StoreU(vec2_4, quarter_tag, buf + stride);
hn::StoreU(vec3_4, quarter_tag, buf + 2 * stride);
hn::StoreU(vec4_4, quarter_tag, buf + 3 * stride);
}
template <typename D>
HWY_ATTR HWY_INLINE hn::VFromD<D> LoadUnaligned2x8(D tag16, const uint8_t *buf,
ptrdiff_t stride) {
hn::Rebind<uint8_t, D> tag8;
constexpr hn::Half<decltype(tag8)> half_tag8;
auto first_half = hn::LoadU(half_tag8, buf);
auto second_half = hn::LoadU(half_tag8, buf + stride);
return hn::PromoteTo(tag16, hn::Combine(tag8, first_half, second_half));
}
template <typename D>
HWY_ATTR HWY_INLINE void StoreUnaligned2x8(D tag, uint8_t *buf,
ptrdiff_t stride,
hn::VFromD<D> &vec) {
(void)tag;
hn::Rebind<uint8_t, D> tag8;
constexpr hn::Half<D> half_tag16;
auto vec_demoted = hn::ReorderDemote2To(tag8, hn::LowerHalf(half_tag16, vec),
hn::UpperHalf(half_tag16, vec));
constexpr hn::Half<decltype(tag8)> half_tag8;
auto vec1_2 = hn::UpperHalf(half_tag8, vec_demoted);
auto vec2_2 = hn::LowerHalf(half_tag8, vec_demoted);
hn::StoreU(vec1_2, half_tag8, buf);
hn::StoreU(vec2_2, half_tag8, buf + stride);
}
template <typename D>
HWY_ATTR HWY_INLINE hn::VFromD<D> LoadUnaligned4x8(D scalable_tag,
const uint8_t *buf,
ptrdiff_t stride) {
hn::Rebind<uint8_t, D> tag8;
constexpr hn::Half<decltype(tag8)> half_tag8;
constexpr hn::Half<decltype(half_tag8)> quarter_tag8;
auto first_quarter = hn::LoadU(quarter_tag8, buf);
auto second_quarter = hn::LoadU(quarter_tag8, buf + stride);
auto third_quarter = hn::LoadU(quarter_tag8, buf + 2 * stride);
auto fourth_quarter = hn::LoadU(quarter_tag8, buf + 3 * stride);
return hn::PromoteTo(
scalable_tag,
hn::Combine(tag8, hn::Combine(half_tag8, first_quarter, second_quarter),
hn::Combine(half_tag8, third_quarter, fourth_quarter)));
}
template <typename D>
HWY_ATTR HWY_INLINE void StoreUnaligned4x8(D tag, uint8_t *buf,
ptrdiff_t stride,
hn::VFromD<D> &vec) {
(void)tag;
hn::Rebind<uint8_t, D> tag8;
constexpr hn::Half<D> half_tag16;
auto vec_demoted = hn::ReorderDemote2To(tag8, hn::LowerHalf(half_tag16, vec),
hn::UpperHalf(half_tag16, vec));
constexpr hn::Half<decltype(tag8)> half_tag8;
constexpr hn::Half<decltype(half_tag8)> quarter_tag8;
auto vec1_2 = hn::UpperHalf(half_tag8, vec_demoted);
auto vec2_2 = hn::LowerHalf(half_tag8, vec_demoted);
auto vec1_4 = hn::UpperHalf(quarter_tag8, vec1_2);
auto vec2_4 = hn::LowerHalf(quarter_tag8, vec1_2);
auto vec3_4 = hn::UpperHalf(quarter_tag8, vec2_2);
auto vec4_4 = hn::LowerHalf(quarter_tag8, vec2_2);
hn::StoreU(vec1_4, quarter_tag8, buf);
hn::StoreU(vec2_4, quarter_tag8, buf + stride);
hn::StoreU(vec3_4, quarter_tag8, buf + 2 * stride);
hn::StoreU(vec4_4, quarter_tag8, buf + 3 * stride);
}
HWY_ATTR inline void ConvolveHoriz2Tap(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int w, int h) {
const bool can_use_pareto_optimal_2tap =
(h == 32 && (w == 16 || w == 32 || w == 64)) && (filter_x[3] % 2 == 0) &&
(filter_x[4] % 2 == 0);
if (can_use_pareto_optimal_2tap) {
if (w == 16) {
constexpr hn::CappedTag<uint8_t, 16> u8_16_tag;
constexpr hn::CappedTag<int8_t, 16> i8_16_tag;
constexpr hn::CappedTag<int16_t, 8> i16_8_tag;
const auto shuf34_16 = hn::Dup128VecFromValues(
u8_16_tag, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const auto coeff34_16 =
hn::Dup128VecFromValues(i8_16_tag, c3, c4, c3, c4, c3, c4, c3, c4, c3,
c4, c3, c4, c3, c4, c3, c4);
const auto round_vec_16 = hn::Set(i16_8_tag, 1 << (FILTER_BITS - 2));
for (int y = 0; y < h; ++y) {
const uint8_t *src_row = src + y * src_stride;
uint8_t *dst_row = dst + y * dst_stride;
auto v0 = hn::LoadU(u8_16_tag, src_row + 0);
auto v8 = hn::LoadU(u8_16_tag, src_row + 8);
auto p34_0 = hn::TableLookupBytes(v0, shuf34_16);
auto res0 = hn::ShiftRightSame(
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p34_0, coeff34_16),
round_vec_16),
FILTER_BITS - 1);
auto p34_8 = hn::TableLookupBytes(v8, shuf34_16);
auto res8 = hn::ShiftRightSame(
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p34_8, coeff34_16),
round_vec_16),
FILTER_BITS - 1);
auto packed = hn::ReorderDemote2To(u8_16_tag, res0, res8);
hn::StoreU(packed, u8_16_tag, dst_row);
}
} else {
constexpr hn::CappedTag<uint8_t, 16> u8_16_tag;
constexpr hn::CappedTag<uint8_t, 32> u8_32_tag;
constexpr hn::CappedTag<int8_t, 16> i8_16_tag;
constexpr hn::CappedTag<int8_t, 32> i8_32_tag;
constexpr hn::CappedTag<int16_t, 16> i16_16_tag;
const auto shuf34_16 = hn::Dup128VecFromValues(
u8_16_tag, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const auto shuf34_32 = hn::Combine(u8_32_tag, shuf34_16, shuf34_16);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const auto coeff34_16 =
hn::Dup128VecFromValues(i8_16_tag, c3, c4, c3, c4, c3, c4, c3, c4, c3,
c4, c3, c4, c3, c4, c3, c4);
const auto coeff34_32 = hn::Combine(i8_32_tag, coeff34_16, coeff34_16);
const auto round_vec_32 = hn::Set(i16_16_tag, 1 << (FILTER_BITS - 2));
for (int y = 0; y < h; ++y) {
const uint8_t *src_row = src + y * src_stride;
uint8_t *dst_row = dst + y * dst_stride;
for (int j = 0; j < w; j += 32) {
auto v_curr = hn::LoadU(u8_32_tag, src_row + j + 0);
auto v8 = hn::LoadU(u8_32_tag, src_row + j + 8);
auto p34_0 = hn::TableLookupBytes(v_curr, shuf34_32);
auto res0 = hn::ShiftRightSame(
hn::Add(hn::SatWidenMulPairwiseAdd(i16_16_tag, p34_0, coeff34_32),
round_vec_32),
FILTER_BITS - 1);
auto p34_8 = hn::TableLookupBytes(v8, shuf34_32);
auto res8 = hn::ShiftRightSame(
hn::Add(hn::SatWidenMulPairwiseAdd(i16_16_tag, p34_8, coeff34_32),
round_vec_32),
FILTER_BITS - 1);
constexpr hn::CappedTag<uint8_t, 8> u8_8_tag;
auto demoted0 = hn::DemoteTo(u8_16_tag, res0);
auto demoted8 = hn::DemoteTo(u8_16_tag, res8);
auto p0_15 = hn::Combine(u8_16_tag, hn::LowerHalf(u8_8_tag, demoted8),
hn::LowerHalf(u8_8_tag, demoted0));
auto p16_31 =
hn::Combine(u8_16_tag, hn::UpperHalf(u8_8_tag, demoted8),
hn::UpperHalf(u8_8_tag, demoted0));
hn::StoreU(p0_15, u8_16_tag, dst_row + j);
hn::StoreU(p16_31, u8_16_tag, dst_row + j + 16);
}
}
}
return;
}
const bool can_use_optimized_path =
(w <= 32) && (filter_x[3] % 2 == 0) && (filter_x[4] % 2 == 0);
if (can_use_optimized_path) {
hn::CappedTag<uint8_t, 16> tag8_16;
hn::CappedTag<int8_t, 16> tag_i8;
hn::CappedTag<int16_t, 8> tag16_8;
hn::CappedTag<uint8_t, 8> tag8_8;
hn::CappedTag<uint8_t, 4> tag8_4;
const auto bias_val = hn::Set(tag16_8, 1 << (FILTER_BITS - 2));
const auto shuffle_mask = hn::Dup128VecFromValues(
tag8_16, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const auto coeff34 = hn::Dup128VecFromValues(
tag_i8, c3, c4, c3, c4, c3, c4, c3, c4, c3, c4, c3, c4, c3, c4, c3, c4);
auto sum_2tap = [&](const uint8_t *s, int offset) {
return ComputeChunkSum2Tap(tag16_8, tag8_16, s, offset, shuffle_mask,
coeff34);
};
if (w == 4) {
while (h >= 2) {
Process2RowsChunk<4>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_2tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<4>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_2tap);
}
} else if (w == 8) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_2tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_2tap);
}
} else if (w == 16) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_2tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
8, bias_val, sum_2tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_2tap);
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 8, bias_val, sum_2tap);
}
} else if (w == 32) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_2tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
8, bias_val, sum_2tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
16, bias_val, sum_2tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
24, bias_val, sum_2tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
for (int j = 0; j < 32; j += 8) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, j, bias_val, sum_2tap);
}
}
}
} else {
hn::ScalableTag<int16_t> mul_tag;
hn::Rebind<uint8_t, decltype(mul_tag)> pixel_tag;
auto filter_0 = hn::Set(mul_tag, filter_x[3]);
auto filter_1 = hn::Set(mul_tag, filter_x[4]);
auto vw = hn::Lanes(mul_tag);
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += vw) {
auto src0 = hn::PromoteTo(mul_tag, hn::LoadU(pixel_tag, &src[j]));
auto src1 = hn::PromoteTo(mul_tag, hn::LoadU(pixel_tag, &src[j + 1]));
auto mulv = hn::RoundingShiftRight<FILTER_BITS>(src0 * filter_0 +
src1 * filter_1);
auto mulv_demoted = hn::DemoteTo(pixel_tag, mulv);
if (j + static_cast<int>(vw) > w) {
hn::StoreN(mulv_demoted, pixel_tag, &dst[j], w - j);
} else {
hn::StoreU(mulv_demoted, pixel_tag, &dst[j]);
}
}
src += src_stride;
dst += dst_stride;
}
}
}
template <typename D, typename DFilter>
HWY_ATTR HWY_INLINE hn::VFromD<D> Convolve4_8(
D tag16, DFilter tag_filter, hn::VFromD<D> &s0, hn::VFromD<D> &s1,
hn::VFromD<D> &s2, hn::VFromD<D> &s3, hn::VFromD<DFilter> &filter) {
(void)tag_filter;
auto mul0 = hn::Mul(s0, hn::Set(tag16, hn::ExtractLane(filter, 0)));
auto mul1 = hn::Mul(s1, hn::Set(tag16, hn::ExtractLane(filter, 1)));
auto mul2 = hn::Mul(s2, hn::Set(tag16, hn::ExtractLane(filter, 2)));
auto mul3 = hn::Mul(s3, hn::Set(tag16, hn::ExtractLane(filter, 3)));
auto res = mul0 + mul1 + mul2 + mul3;
// Shift (FILTER_BITS - 1) because filter values were halved.
return hn::RoundingShiftRight<FILTER_BITS - 1>(res);
}
HWY_ATTR inline void ConvolveHoriz4Tap(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int w, int h) {
const bool can_use_pareto_optimal_4tap =
(h == 32 && (w == 16 || w == 32 || w == 64)) && (filter_x[2] % 2 == 0) &&
(filter_x[3] % 2 == 0) && (filter_x[4] % 2 == 0) &&
(filter_x[5] % 2 == 0);
if (can_use_pareto_optimal_4tap) {
if (w == 16) {
constexpr hn::CappedTag<uint8_t, 16> u8_16_tag;
constexpr hn::CappedTag<int8_t, 16> i8_16_tag;
constexpr hn::CappedTag<int16_t, 8> i16_8_tag;
const auto shuf01_16 = hn::Dup128VecFromValues(
u8_16_tag, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const auto shuf23_16 = hn::Dup128VecFromValues(
u8_16_tag, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10);
const int8_t c2 = static_cast<int8_t>(filter_x[2] / 2);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const int8_t c5 = static_cast<int8_t>(filter_x[5] / 2);
const auto coeff23_16 =
hn::Dup128VecFromValues(i8_16_tag, c2, c3, c2, c3, c2, c3, c2, c3, c2,
c3, c2, c3, c2, c3, c2, c3);
const auto coeff45_16 =
hn::Dup128VecFromValues(i8_16_tag, c4, c5, c4, c5, c4, c5, c4, c5, c4,
c5, c4, c5, c4, c5, c4, c5);
const auto round_vec_16 = hn::Set(i16_8_tag, 1 << (FILTER_BITS - 2));
for (int y = 0; y < h; ++y) {
const uint8_t *src_row = src + y * src_stride;
uint8_t *dst_row = dst + y * dst_stride;
auto v0 = hn::LoadU(u8_16_tag, src_row + 0);
auto v8 = hn::LoadU(u8_16_tag, src_row + 8);
auto p01_0 = hn::TableLookupBytes(v0, shuf01_16);
auto p23_0 = hn::TableLookupBytes(v0, shuf23_16);
auto sum0 =
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p01_0, coeff23_16),
hn::SatWidenMulPairwiseAdd(i16_8_tag, p23_0, coeff45_16));
auto res0 =
hn::ShiftRightSame(hn::Add(sum0, round_vec_16), FILTER_BITS - 1);
auto p01_8 = hn::TableLookupBytes(v8, shuf01_16);
auto p23_8 = hn::TableLookupBytes(v8, shuf23_16);
auto sum0_8 =
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p01_8, coeff23_16),
hn::SatWidenMulPairwiseAdd(i16_8_tag, p23_8, coeff45_16));
auto res8 =
hn::ShiftRightSame(hn::Add(sum0_8, round_vec_16), FILTER_BITS - 1);
auto packed = hn::ReorderDemote2To(u8_16_tag, res0, res8);
hn::StoreU(packed, u8_16_tag, dst_row);
}
} else {
constexpr hn::CappedTag<uint8_t, 16> u8_16_tag;
constexpr hn::CappedTag<uint8_t, 32> u8_32_tag;
constexpr hn::CappedTag<int8_t, 16> i8_16_tag;
constexpr hn::CappedTag<int8_t, 32> i8_32_tag;
constexpr hn::CappedTag<int16_t, 16> i16_16_tag;
const auto shuf01_16 = hn::Dup128VecFromValues(
u8_16_tag, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const auto shuf23_16 = hn::Dup128VecFromValues(
u8_16_tag, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10);
const auto shuf01_32 = hn::Combine(u8_32_tag, shuf01_16, shuf01_16);
const auto shuf23_32 = hn::Combine(u8_32_tag, shuf23_16, shuf23_16);
const int8_t c2 = static_cast<int8_t>(filter_x[2] / 2);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const int8_t c5 = static_cast<int8_t>(filter_x[5] / 2);
const auto coeff23_16 =
hn::Dup128VecFromValues(i8_16_tag, c2, c3, c2, c3, c2, c3, c2, c3, c2,
c3, c2, c3, c2, c3, c2, c3);
const auto coeff45_16 =
hn::Dup128VecFromValues(i8_16_tag, c4, c5, c4, c5, c4, c5, c4, c5, c4,
c5, c4, c5, c4, c5, c4, c5);
const auto coeff23_32 = hn::Combine(i8_32_tag, coeff23_16, coeff23_16);
const auto coeff45_32 = hn::Combine(i8_32_tag, coeff45_16, coeff45_16);
const auto round_vec_32 = hn::Set(i16_16_tag, 1 << (FILTER_BITS - 2));
for (int y = 0; y < h; ++y) {
const uint8_t *src_row = src + y * src_stride;
uint8_t *dst_row = dst + y * dst_stride;
for (int j = 0; j < w; j += 32) {
auto v_curr = hn::LoadU(u8_32_tag, src_row + j + 0);
auto v8 = hn::LoadU(u8_32_tag, src_row + j + 8);
auto p01_0 = hn::TableLookupBytes(v_curr, shuf01_32);
auto p23_0 = hn::TableLookupBytes(v_curr, shuf23_32);
auto sum0 = hn::Add(
hn::SatWidenMulPairwiseAdd(i16_16_tag, p01_0, coeff23_32),
hn::SatWidenMulPairwiseAdd(i16_16_tag, p23_0, coeff45_32));
auto res0 =
hn::ShiftRightSame(hn::Add(sum0, round_vec_32), FILTER_BITS - 1);
auto p01_8 = hn::TableLookupBytes(v8, shuf01_32);
auto p23_8 = hn::TableLookupBytes(v8, shuf23_32);
auto sum0_8 = hn::Add(
hn::SatWidenMulPairwiseAdd(i16_16_tag, p01_8, coeff23_32),
hn::SatWidenMulPairwiseAdd(i16_16_tag, p23_8, coeff45_32));
auto res8 = hn::ShiftRightSame(hn::Add(sum0_8, round_vec_32),
FILTER_BITS - 1);
constexpr hn::CappedTag<uint8_t, 8> u8_8_tag;
auto demoted0 = hn::DemoteTo(u8_16_tag, res0);
auto demoted8 = hn::DemoteTo(u8_16_tag, res8);
auto p0_15 = hn::Combine(u8_16_tag, hn::LowerHalf(u8_8_tag, demoted8),
hn::LowerHalf(u8_8_tag, demoted0));
auto p16_31 =
hn::Combine(u8_16_tag, hn::UpperHalf(u8_8_tag, demoted8),
hn::UpperHalf(u8_8_tag, demoted0));
hn::StoreU(p0_15, u8_16_tag, dst_row + j);
hn::StoreU(p16_31, u8_16_tag, dst_row + j + 16);
}
}
}
return;
}
const bool can_use_optimized_path =
(w <= 32) && (filter_x[2] % 2 == 0) && (filter_x[3] % 2 == 0) &&
(filter_x[4] % 2 == 0) && (filter_x[5] % 2 == 0);
if (can_use_optimized_path) {
hn::CappedTag<uint8_t, 16> tag8_16;
hn::CappedTag<int8_t, 16> tag_i8;
hn::CappedTag<int16_t, 8> tag16_8;
hn::CappedTag<uint8_t, 8> tag8_8;
hn::CappedTag<uint8_t, 4> tag8_4;
const auto bias_val = hn::Set(tag16_8, 1 << (FILTER_BITS - 2));
const auto shuffle_mask = hn::Dup128VecFromValues(
tag8_16, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const int8_t c2 = static_cast<int8_t>(filter_x[2] / 2);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const int8_t c5 = static_cast<int8_t>(filter_x[5] / 2);
const auto coeff23 = hn::Dup128VecFromValues(
tag_i8, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3);
const auto coeff45 = hn::Dup128VecFromValues(
tag_i8, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5);
auto sum_4tap = [&](const uint8_t *s, int offset) {
return ComputeChunkSum4Tap(tag16_8, tag8_16, s, offset, shuffle_mask,
coeff23, coeff45);
};
if (w == 4) {
while (h >= 2) {
Process2RowsChunk<4>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_4tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<4>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_4tap);
}
} else if (w == 8) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_4tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_4tap);
}
} else if (w == 16) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_4tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
8, bias_val, sum_4tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_4tap);
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 8, bias_val, sum_4tap);
}
} else if (w == 32) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_4tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
8, bias_val, sum_4tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
16, bias_val, sum_4tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
24, bias_val, sum_4tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
for (int j = 0; j < 32; j += 8) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, j, bias_val, sum_4tap);
}
}
}
} else {
hn::ScalableTag<int16_t> mul_tag;
hn::Rebind<uint8_t, decltype(mul_tag)> pixel_tag;
hn::CappedTag<int16_t, 4> filter_tag;
auto f_vec = hn::LoadU(filter_tag, filter_x + 2);
f_vec = hn::ShiftRight<1>(f_vec);
auto vw = hn::Lanes(mul_tag);
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += vw) {
auto src0 = hn::PromoteTo(mul_tag, hn::LoadU(pixel_tag, &src[j]));
auto src1 = hn::PromoteTo(mul_tag, hn::LoadU(pixel_tag, &src[j + 1]));
auto src2 = hn::PromoteTo(mul_tag, hn::LoadU(pixel_tag, &src[j + 2]));
auto src3 = hn::PromoteTo(mul_tag, hn::LoadU(pixel_tag, &src[j + 3]));
auto result =
Convolve4_8(mul_tag, filter_tag, src0, src1, src2, src3, f_vec);
auto result_demoted = hn::DemoteTo(pixel_tag, result);
if (j + static_cast<int>(vw) > w) {
hn::StoreN(result_demoted, pixel_tag, &dst[j], w - j);
} else {
hn::StoreU(result_demoted, pixel_tag, &dst[j]);
}
}
src += src_stride;
dst += dst_stride;
}
}
}
template <typename D, typename DFilter>
HWY_ATTR HWY_INLINE hn::VFromD<D> Convolve8_8(
D tag16, DFilter tag_filter, hn::VFromD<D> &s0, hn::VFromD<D> &s1,
hn::VFromD<D> &s2, hn::VFromD<D> &s3, hn::VFromD<D> &s4, hn::VFromD<D> &s5,
hn::VFromD<D> &s6, hn::VFromD<D> &s7, hn::VFromD<DFilter> &filter) {
(void)tag_filter;
auto filter_0 = hn::ExtractLane(filter, 0);
auto filter_1 = hn::ExtractLane(filter, 1);
auto filter_2 = hn::ExtractLane(filter, 2);
auto filter_3 = hn::ExtractLane(filter, 3);
auto filter_4 = hn::ExtractLane(filter, 4);
auto filter_5 = hn::ExtractLane(filter, 5);
auto filter_6 = hn::ExtractLane(filter, 6);
auto filter_7 = hn::ExtractLane(filter, 7);
auto mul0 = hn::Mul(s0, hn::Set(tag16, filter_0));
auto mul1 = hn::Mul(s1, hn::Set(tag16, filter_1));
auto mul2 = hn::Mul(s2, hn::Set(tag16, filter_2));
auto mul3 = hn::Mul(s3, hn::Set(tag16, filter_3));
auto mul4 = hn::Mul(s4, hn::Set(tag16, filter_4));
auto mul5 = hn::Mul(s5, hn::Set(tag16, filter_5));
auto mul6 = hn::Mul(s6, hn::Set(tag16, filter_6));
auto mul7 = hn::Mul(s7, hn::Set(tag16, filter_7));
auto res = mul0 + mul1 + mul2 + mul3 + mul4 + mul5 + mul6 + mul7;
// Shift (FILTER_BITS - 1) because filter values were halved.
return hn::RoundingShiftRight<FILTER_BITS - 1>(res);
}
HWY_ATTR inline void ConvolveHoriz8Tap(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int w, int h) {
const bool can_use_pareto_optimal_8tap =
(h == 32 && (w == 16 || w == 32 || w == 64)) && (filter_x[0] % 2 == 0) &&
(filter_x[1] % 2 == 0) && (filter_x[2] % 2 == 0) &&
(filter_x[3] % 2 == 0) && (filter_x[4] % 2 == 0) &&
(filter_x[5] % 2 == 0) && (filter_x[6] % 2 == 0) &&
(filter_x[7] % 2 == 0);
if (can_use_pareto_optimal_8tap) {
if (w == 16) {
constexpr hn::CappedTag<uint8_t, 16> u8_16_tag;
constexpr hn::CappedTag<int8_t, 16> i8_16_tag;
constexpr hn::CappedTag<int16_t, 8> i16_8_tag;
// Construct shuffles locally (no filt_global load)
const auto shuf01_16 = hn::Dup128VecFromValues(
u8_16_tag, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const auto shuf23_16 = hn::Dup128VecFromValues(
u8_16_tag, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10);
const auto shuf45_16 = hn::Dup128VecFromValues(
u8_16_tag, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12);
const auto shuf67_16 = hn::Dup128VecFromValues(
u8_16_tag, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14);
const int8_t c0 = static_cast<int8_t>(filter_x[0] / 2);
const int8_t c1 = static_cast<int8_t>(filter_x[1] / 2);
const int8_t c2 = static_cast<int8_t>(filter_x[2] / 2);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const int8_t c5 = static_cast<int8_t>(filter_x[5] / 2);
const int8_t c6 = static_cast<int8_t>(filter_x[6] / 2);
const int8_t c7 = static_cast<int8_t>(filter_x[7] / 2);
const auto coeff01_16 =
hn::Dup128VecFromValues(i8_16_tag, c0, c1, c0, c1, c0, c1, c0, c1, c0,
c1, c0, c1, c0, c1, c0, c1);
const auto coeff23_16 =
hn::Dup128VecFromValues(i8_16_tag, c2, c3, c2, c3, c2, c3, c2, c3, c2,
c3, c2, c3, c2, c3, c2, c3);
const auto coeff45_16 =
hn::Dup128VecFromValues(i8_16_tag, c4, c5, c4, c5, c4, c5, c4, c5, c4,
c5, c4, c5, c4, c5, c4, c5);
const auto coeff67_16 =
hn::Dup128VecFromValues(i8_16_tag, c6, c7, c6, c7, c6, c7, c6, c7, c6,
c7, c6, c7, c6, c7, c6, c7);
const auto round_vec_16 = hn::Set(i16_8_tag, 1 << (FILTER_BITS - 2));
for (int y = 0; y < h; ++y) {
const uint8_t *src_row = src + y * src_stride;
uint8_t *dst_row = dst + y * dst_stride;
auto v0 = hn::LoadU(u8_16_tag, src_row + 0);
auto v8 = hn::LoadU(u8_16_tag, src_row + 8);
auto p01_0 = hn::TableLookupBytes(v0, shuf01_16);
auto p23_0 = hn::TableLookupBytes(v0, shuf23_16);
auto p45_0 = hn::TableLookupBytes(v0, shuf45_16);
auto p67_0 = hn::TableLookupBytes(v0, shuf67_16);
auto sum0 =
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p01_0, coeff01_16),
hn::SatWidenMulPairwiseAdd(i16_8_tag, p23_0, coeff23_16));
auto sum1 =
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p45_0, coeff45_16),
hn::SatWidenMulPairwiseAdd(i16_8_tag, p67_0, coeff67_16));
auto res0 = hn::ShiftRightSame(
hn::Add(hn::Add(sum0, sum1), round_vec_16), FILTER_BITS - 1);
auto p01_8 = hn::TableLookupBytes(v8, shuf01_16);
auto p23_8 = hn::TableLookupBytes(v8, shuf23_16);
auto p45_8 = hn::TableLookupBytes(v8, shuf45_16);
auto p67_8 = hn::TableLookupBytes(v8, shuf67_16);
auto sum0_8 =
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p01_8, coeff01_16),
hn::SatWidenMulPairwiseAdd(i16_8_tag, p23_8, coeff23_16));
auto sum1_8 =
hn::Add(hn::SatWidenMulPairwiseAdd(i16_8_tag, p45_8, coeff45_16),
hn::SatWidenMulPairwiseAdd(i16_8_tag, p67_8, coeff67_16));
auto res8 = hn::ShiftRightSame(
hn::Add(hn::Add(sum0_8, sum1_8), round_vec_16), FILTER_BITS - 1);
auto packed = hn::ReorderDemote2To(u8_16_tag, res0, res8);
hn::StoreU(packed, u8_16_tag, dst_row);
}
} else {
constexpr hn::CappedTag<uint8_t, 16> u8_16_tag;
constexpr hn::CappedTag<uint8_t, 32> u8_32_tag;
constexpr hn::CappedTag<int8_t, 16> i8_16_tag;
constexpr hn::CappedTag<int8_t, 32> i8_32_tag;
constexpr hn::CappedTag<int16_t, 16> i16_16_tag;
const auto shuf01_16 = hn::Dup128VecFromValues(
u8_16_tag, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const auto shuf23_16 = hn::Dup128VecFromValues(
u8_16_tag, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10);
const auto shuf45_16 = hn::Dup128VecFromValues(
u8_16_tag, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12);
const auto shuf67_16 = hn::Dup128VecFromValues(
u8_16_tag, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14);
const auto shuf01_32 = hn::Combine(u8_32_tag, shuf01_16, shuf01_16);
const auto shuf23_32 = hn::Combine(u8_32_tag, shuf23_16, shuf23_16);
const auto shuf45_32 = hn::Combine(u8_32_tag, shuf45_16, shuf45_16);
const auto shuf67_32 = hn::Combine(u8_32_tag, shuf67_16, shuf67_16);
const int8_t c0 = static_cast<int8_t>(filter_x[0] / 2);
const int8_t c1 = static_cast<int8_t>(filter_x[1] / 2);
const int8_t c2 = static_cast<int8_t>(filter_x[2] / 2);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const int8_t c5 = static_cast<int8_t>(filter_x[5] / 2);
const int8_t c6 = static_cast<int8_t>(filter_x[6] / 2);
const int8_t c7 = static_cast<int8_t>(filter_x[7] / 2);
const auto coeff01_16 =
hn::Dup128VecFromValues(i8_16_tag, c0, c1, c0, c1, c0, c1, c0, c1, c0,
c1, c0, c1, c0, c1, c0, c1);
const auto coeff23_16 =
hn::Dup128VecFromValues(i8_16_tag, c2, c3, c2, c3, c2, c3, c2, c3, c2,
c3, c2, c3, c2, c3, c2, c3);
const auto coeff45_16 =
hn::Dup128VecFromValues(i8_16_tag, c4, c5, c4, c5, c4, c5, c4, c5, c4,
c5, c4, c5, c4, c5, c4, c5);
const auto coeff67_16 =
hn::Dup128VecFromValues(i8_16_tag, c6, c7, c6, c7, c6, c7, c6, c7, c6,
c7, c6, c7, c6, c7, c6, c7);
const auto coeff01_32 = hn::Combine(i8_32_tag, coeff01_16, coeff01_16);
const auto coeff23_32 = hn::Combine(i8_32_tag, coeff23_16, coeff23_16);
const auto coeff45_32 = hn::Combine(i8_32_tag, coeff45_16, coeff45_16);
const auto coeff67_32 = hn::Combine(i8_32_tag, coeff67_16, coeff67_16);
const auto round_vec_32 = hn::Set(i16_16_tag, 1 << (FILTER_BITS - 2));
for (int y = 0; y < h; ++y) {
const uint8_t *src_row = src + y * src_stride;
uint8_t *dst_row = dst + y * dst_stride;
for (int j = 0; j < w; j += 32) {
auto v_curr = hn::LoadU(u8_32_tag, src_row + j + 0);
auto v8 = hn::LoadU(u8_32_tag, src_row + j + 8);
auto p01_0 = hn::TableLookupBytes(v_curr, shuf01_32);
auto p23_0 = hn::TableLookupBytes(v_curr, shuf23_32);
auto p45_0 = hn::TableLookupBytes(v_curr, shuf45_32);
auto p67_0 = hn::TableLookupBytes(v_curr, shuf67_32);
auto sum0 = hn::Add(
hn::SatWidenMulPairwiseAdd(i16_16_tag, p01_0, coeff01_32),
hn::SatWidenMulPairwiseAdd(i16_16_tag, p23_0, coeff23_32));
auto sum1 = hn::Add(
hn::SatWidenMulPairwiseAdd(i16_16_tag, p45_0, coeff45_32),
hn::SatWidenMulPairwiseAdd(i16_16_tag, p67_0, coeff67_32));
auto res0 = hn::ShiftRightSame(
hn::Add(hn::Add(sum0, sum1), round_vec_32), FILTER_BITS - 1);
auto p01_8 = hn::TableLookupBytes(v8, shuf01_32);
auto p23_8 = hn::TableLookupBytes(v8, shuf23_32);
auto p45_8 = hn::TableLookupBytes(v8, shuf45_32);
auto p67_8 = hn::TableLookupBytes(v8, shuf67_32);
auto sum0_8 = hn::Add(
hn::SatWidenMulPairwiseAdd(i16_16_tag, p01_8, coeff01_32),
hn::SatWidenMulPairwiseAdd(i16_16_tag, p23_8, coeff23_32));
auto sum1_8 = hn::Add(
hn::SatWidenMulPairwiseAdd(i16_16_tag, p45_8, coeff45_32),
hn::SatWidenMulPairwiseAdd(i16_16_tag, p67_8, coeff67_32));
auto res8 = hn::ShiftRightSame(
hn::Add(hn::Add(sum0_8, sum1_8), round_vec_32), FILTER_BITS - 1);
constexpr hn::CappedTag<uint8_t, 8> u8_8_tag;
auto demoted0 = hn::DemoteTo(u8_16_tag, res0);
auto demoted8 = hn::DemoteTo(u8_16_tag, res8);
auto p0_15 = hn::Combine(u8_16_tag, hn::LowerHalf(u8_8_tag, demoted8),
hn::LowerHalf(u8_8_tag, demoted0));
auto p16_31 =
hn::Combine(u8_16_tag, hn::UpperHalf(u8_8_tag, demoted8),
hn::UpperHalf(u8_8_tag, demoted0));
hn::StoreU(p0_15, u8_16_tag, dst_row + j);
hn::StoreU(p16_31, u8_16_tag, dst_row + j + 16);
}
}
}
return;
}
const bool can_use_optimized_path =
(w <= 32) && (filter_x[0] % 2 == 0) && (filter_x[1] % 2 == 0) &&
(filter_x[2] % 2 == 0) && (filter_x[3] % 2 == 0) &&
(filter_x[4] % 2 == 0) && (filter_x[5] % 2 == 0) &&
(filter_x[6] % 2 == 0) && (filter_x[7] % 2 == 0);
if (can_use_optimized_path) {
hn::CappedTag<uint8_t, 16> tag8_16;
hn::CappedTag<int8_t, 16> tag_i8;
hn::CappedTag<int16_t, 8> tag16_8;
hn::CappedTag<uint8_t, 8> tag8_8;
hn::CappedTag<uint8_t, 4> tag8_4;
const auto bias_val = hn::Set(tag16_8, 1 << (FILTER_BITS - 2));
const auto shuffle_mask = hn::Dup128VecFromValues(
tag8_16, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
const int8_t c0 = static_cast<int8_t>(filter_x[0] / 2);
const int8_t c1 = static_cast<int8_t>(filter_x[1] / 2);
const int8_t c2 = static_cast<int8_t>(filter_x[2] / 2);
const int8_t c3 = static_cast<int8_t>(filter_x[3] / 2);
const int8_t c4 = static_cast<int8_t>(filter_x[4] / 2);
const int8_t c5 = static_cast<int8_t>(filter_x[5] / 2);
const int8_t c6 = static_cast<int8_t>(filter_x[6] / 2);
const int8_t c7 = static_cast<int8_t>(filter_x[7] / 2);
const auto coeff01 = hn::Dup128VecFromValues(
tag_i8, c0, c1, c0, c1, c0, c1, c0, c1, c0, c1, c0, c1, c0, c1, c0, c1);
const auto coeff23 = hn::Dup128VecFromValues(
tag_i8, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3, c2, c3);
const auto coeff45 = hn::Dup128VecFromValues(
tag_i8, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5, c4, c5);
const auto coeff67 = hn::Dup128VecFromValues(
tag_i8, c6, c7, c6, c7, c6, c7, c6, c7, c6, c7, c6, c7, c6, c7, c6, c7);
auto sum_8tap = [&](const uint8_t *s, int offset) {
return ComputeChunkSum8Tap(tag16_8, tag8_16, s, offset, shuffle_mask,
coeff01, coeff23, coeff45, coeff67);
};
if (w == 4) {
while (h >= 2) {
Process2RowsChunk<4>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_8tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<4>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_8tap);
}
} else if (w == 8) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_8tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_8tap);
}
} else if (w == 16) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_8tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
8, bias_val, sum_8tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 0, bias_val, sum_8tap);
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, 8, bias_val, sum_8tap);
}
} else if (w == 32) {
while (h >= 2) {
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
0, bias_val, sum_8tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
8, bias_val, sum_8tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
16, bias_val, sum_8tap);
Process2RowsChunk<8>(tag8_8, tag8_4, src, src_stride, dst, dst_stride,
24, bias_val, sum_8tap);
src += 2 * src_stride;
dst += 2 * dst_stride;
h -= 2;
}
if (h & 1) {
for (int j = 0; j < 32; j += 8) {
Process1RowChunk<8>(tag8_8, tag8_4, src, dst, j, bias_val, sum_8tap);
}
}
}
} else {
hn::CappedTag<int16_t, 8> filter_tag;
auto f_vec = hn::LoadU(filter_tag, filter_x);
f_vec = hn::ShiftRight<1>(f_vec);
hn::ScalableTag<int16_t> mul_tag;
hn::Rebind<uint8_t, decltype(mul_tag)> pixel_tag;
auto vw = hn::Lanes(mul_tag);
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; j += vw) {
auto s0 = hn::LoadU(pixel_tag, &src[j]);
auto s1 = hn::LoadU(pixel_tag, &src[j + 1]);
auto s2 = hn::LoadU(pixel_tag, &src[j + 2]);
auto s3 = hn::LoadU(pixel_tag, &src[j + 3]);
auto s4 = hn::LoadU(pixel_tag, &src[j + 4]);
auto s5 = hn::LoadU(pixel_tag, &src[j + 5]);
auto s6 = hn::LoadU(pixel_tag, &src[j + 6]);
auto s7 = hn::LoadU(pixel_tag, &src[j + 7]);
auto src0 = hn::PromoteTo(mul_tag, s0);
auto src1 = hn::PromoteTo(mul_tag, s1);
auto src2 = hn::PromoteTo(mul_tag, s2);
auto src3 = hn::PromoteTo(mul_tag, s3);
auto src4 = hn::PromoteTo(mul_tag, s4);
auto src5 = hn::PromoteTo(mul_tag, s5);
auto src6 = hn::PromoteTo(mul_tag, s6);
auto src7 = hn::PromoteTo(mul_tag, s7);
auto result = Convolve8_8(mul_tag, filter_tag, src0, src1, src2, src3,
src4, src5, src6, src7, f_vec);
auto result_demoted = hn::DemoteTo(pixel_tag, result);
if (j + static_cast<int>(vw) > w) {
hn::StoreN(result_demoted, pixel_tag, &dst[j], w - j);
} else {
hn::StoreU(result_demoted, pixel_tag, &dst[j]);
}
}
src += src_stride;
dst += dst_stride;
}
}
}
HWY_ATTR inline void ConvolveVert2Tap(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_y, int w, int h) {
hn::CappedTag<int16_t, 16> tag16;
hn::Rebind<uint8_t, decltype(tag16)> pixel_tag;
auto f0 = hn::Set(tag16, filter_y[3]);
auto f1 = hn::Set(tag16, filter_y[4]);
auto round_offset = hn::Set(tag16, 1 << (FILTER_BITS - 1));
if (w == 4) {
for (int y = 0; y < h; y += 4) {
auto s0 = LoadUnaligned4x4(tag16, src + y * src_stride, src_stride);
auto s1 = LoadUnaligned4x4(tag16, src + (y + 1) * src_stride, src_stride);
auto res = hn::ShiftRight<FILTER_BITS>(s0 * f0 + s1 * f1 + round_offset);
StoreUnaligned4x4(tag16, dst + y * dst_stride, dst_stride, res);
}
} else if (w == 8) {
for (int y = 0; y < h; y += 4) {
auto s0 = LoadUnaligned2x8(tag16, src + y * src_stride, src_stride);
auto s1 = LoadUnaligned2x8(tag16, src + (y + 1) * src_stride, src_stride);
auto s2 = LoadUnaligned2x8(tag16, src + (y + 2) * src_stride, src_stride);
auto s3 = LoadUnaligned2x8(tag16, src + (y + 3) * src_stride, src_stride);
auto res0 = hn::ShiftRight<FILTER_BITS>(s0 * f0 + s1 * f1 + round_offset);
auto res1 = hn::ShiftRight<FILTER_BITS>(s2 * f0 + s3 * f1 + round_offset);
StoreUnaligned2x8(tag16, dst + y * dst_stride, dst_stride, res0);
StoreUnaligned2x8(tag16, dst + (y + 2) * dst_stride, dst_stride, res1);
}
} else if (w == 16) {
constexpr hn::Half<decltype(tag16)> half_tag16;
auto s0 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src));
for (int y = 0; y < h; y += 4) {
auto s1 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 1) * src_stride));
auto s2 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 2) * src_stride));
auto s3 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 3) * src_stride));
auto s4 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 4) * src_stride));
auto res0 = hn::ShiftRight<FILTER_BITS>(s0 * f0 + s1 * f1 + round_offset);
auto res1 = hn::ShiftRight<FILTER_BITS>(s1 * f0 + s2 * f1 + round_offset);
auto res2 = hn::ShiftRight<FILTER_BITS>(s2 * f0 + s3 * f1 + round_offset);
auto res3 = hn::ShiftRight<FILTER_BITS>(s3 * f0 + s4 * f1 + round_offset);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res0),
hn::UpperHalf(half_tag16, res0)),
pixel_tag, dst + y * dst_stride);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res1),
hn::UpperHalf(half_tag16, res1)),
pixel_tag, dst + (y + 1) * dst_stride);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res2),
hn::UpperHalf(half_tag16, res2)),
pixel_tag, dst + (y + 2) * dst_stride);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res3),
hn::UpperHalf(half_tag16, res3)),
pixel_tag, dst + (y + 3) * dst_stride);
s0 = s4;
}
} else {
hn::ScalableTag<int16_t> mul_tag;
hn::Rebind<uint8_t, decltype(mul_tag)> p_tag;
auto f0_s = hn::Set(mul_tag, filter_y[3]);
auto f1_s = hn::Set(mul_tag, filter_y[4]);
auto round_offset_s = hn::Set(mul_tag, 1 << (FILTER_BITS - 1));
auto vw = hn::Lanes(mul_tag);
for (int x = 0; x < w; x += vw) {
auto s0 = hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x));
for (int y = 0; y < h; ++y) {
auto s1 = hn::PromoteTo(
mul_tag, hn::LoadU(p_tag, src + x + (y + 1) * src_stride));
auto res =
hn::ShiftRight<FILTER_BITS>(s0 * f0_s + s1 * f1_s + round_offset_s);
auto res_demoted = hn::DemoteTo(p_tag, res);
if (x + static_cast<int>(vw) > w) {
hn::StoreN(res_demoted, p_tag, dst + x + y * dst_stride, w - x);
} else {
hn::StoreU(res_demoted, p_tag, dst + x + y * dst_stride);
}
s0 = s1;
}
}
}
}
HWY_ATTR inline void ConvolveVert4Tap(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_y, int w, int h) {
hn::CappedTag<int16_t, 16> tag16;
hn::Rebind<uint8_t, decltype(tag16)> pixel_tag;
auto f0 = hn::Set(tag16, filter_y[2] >> 1);
auto f1 = hn::Set(tag16, filter_y[3] >> 1);
auto f2 = hn::Set(tag16, filter_y[4] >> 1);
auto f3 = hn::Set(tag16, filter_y[5] >> 1);
auto round_offset = hn::Set(tag16, 1 << (FILTER_BITS - 2));
if (w == 4) {
for (int y = 0; y < h; y += 4) {
auto s0 = LoadUnaligned4x4(tag16, src + y * src_stride, src_stride);
auto s1 = LoadUnaligned4x4(tag16, src + (y + 1) * src_stride, src_stride);
auto s2 = LoadUnaligned4x4(tag16, src + (y + 2) * src_stride, src_stride);
auto s3 = LoadUnaligned4x4(tag16, src + (y + 3) * src_stride, src_stride);
auto res = hn::ShiftRight<FILTER_BITS - 1>(s0 * f0 + s1 * f1 + s2 * f2 +
s3 * f3 + round_offset);
StoreUnaligned4x4(tag16, dst + y * dst_stride, dst_stride, res);
}
} else if (w == 8) {
auto s0 = LoadUnaligned2x8(tag16, src + 0 * src_stride, src_stride);
auto s1 = LoadUnaligned2x8(tag16, src + 1 * src_stride, src_stride);
for (int y = 0; y < h; y += 4) {
auto s2 = LoadUnaligned2x8(tag16, src + (y + 2) * src_stride, src_stride);
auto s3 = LoadUnaligned2x8(tag16, src + (y + 3) * src_stride, src_stride);
auto s4 = LoadUnaligned2x8(tag16, src + (y + 4) * src_stride, src_stride);
auto s5 = LoadUnaligned2x8(tag16, src + (y + 5) * src_stride, src_stride);
auto res0 = hn::ShiftRight<FILTER_BITS - 1>(s0 * f0 + s1 * f1 + s2 * f2 +
s3 * f3 + round_offset);
auto res1 = hn::ShiftRight<FILTER_BITS - 1>(s2 * f0 + s3 * f1 + s4 * f2 +
s5 * f3 + round_offset);
StoreUnaligned2x8(tag16, dst + y * dst_stride, dst_stride, res0);
StoreUnaligned2x8(tag16, dst + (y + 2) * dst_stride, dst_stride, res1);
s0 = s4;
s1 = s5;
}
} else if (w == 16) {
constexpr hn::Half<decltype(tag16)> half_tag16;
auto s0 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 0 * src_stride));
auto s1 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 1 * src_stride));
auto s2 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 2 * src_stride));
for (int y = 0; y < h; y += 4) {
auto s3 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 3) * src_stride));
auto s4 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 4) * src_stride));
auto s5 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 5) * src_stride));
auto s6 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 6) * src_stride));
auto res0 = hn::ShiftRight<FILTER_BITS - 1>(s0 * f0 + s1 * f1 + s2 * f2 +
s3 * f3 + round_offset);
auto res1 = hn::ShiftRight<FILTER_BITS - 1>(s1 * f0 + s2 * f1 + s3 * f2 +
s4 * f3 + round_offset);
auto res2 = hn::ShiftRight<FILTER_BITS - 1>(s2 * f0 + s3 * f1 + s4 * f2 +
s5 * f3 + round_offset);
auto res3 = hn::ShiftRight<FILTER_BITS - 1>(s3 * f0 + s4 * f1 + s5 * f2 +
s6 * f3 + round_offset);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res0),
hn::UpperHalf(half_tag16, res0)),
pixel_tag, dst + y * dst_stride);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res1),
hn::UpperHalf(half_tag16, res1)),
pixel_tag, dst + (y + 1) * dst_stride);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res2),
hn::UpperHalf(half_tag16, res2)),
pixel_tag, dst + (y + 2) * dst_stride);
hn::StoreU(
hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res3),
hn::UpperHalf(half_tag16, res3)),
pixel_tag, dst + (y + 3) * dst_stride);
s0 = s4;
s1 = s5;
s2 = s6;
}
} else {
hn::ScalableTag<int16_t> mul_tag;
hn::Rebind<uint8_t, decltype(mul_tag)> p_tag;
auto f0_s = hn::Set(mul_tag, filter_y[2] >> 1);
auto f1_s = hn::Set(mul_tag, filter_y[3] >> 1);
auto f2_s = hn::Set(mul_tag, filter_y[4] >> 1);
auto f3_s = hn::Set(mul_tag, filter_y[5] >> 1);
auto round_offset_s = hn::Set(mul_tag, 1 << (FILTER_BITS - 2));
auto vw = hn::Lanes(mul_tag);
for (int x = 0; x < w; x += vw) {
auto s0 = hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x));
auto s1 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 1 * src_stride));
auto s2 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 2 * src_stride));
for (int y = 0; y < h; ++y) {
auto s3 = hn::PromoteTo(
mul_tag, hn::LoadU(p_tag, src + x + (y + 3) * src_stride));
auto res = hn::ShiftRight<FILTER_BITS - 1>(
s0 * f0_s + s1 * f1_s + s2 * f2_s + s3 * f3_s + round_offset_s);
auto res_demoted = hn::DemoteTo(p_tag, res);
if (x + static_cast<int>(vw) > w) {
hn::StoreN(res_demoted, p_tag, dst + x + y * dst_stride, w - x);
} else {
hn::StoreU(res_demoted, p_tag, dst + x + y * dst_stride);
}
s0 = s1;
s1 = s2;
s2 = s3;
}
}
}
}
HWY_ATTR inline void ConvolveVert8Tap(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_y, int w, int h) {
hn::CappedTag<int16_t, 16> tag16;
hn::Rebind<uint8_t, decltype(tag16)> pixel_tag;
auto f0 = hn::Set(tag16, filter_y[0] >> 1);
auto f1 = hn::Set(tag16, filter_y[1] >> 1);
auto f2 = hn::Set(tag16, filter_y[2] >> 1);
auto f3 = hn::Set(tag16, filter_y[3] >> 1);
auto f4 = hn::Set(tag16, filter_y[4] >> 1);
auto f5 = hn::Set(tag16, filter_y[5] >> 1);
auto f6 = hn::Set(tag16, filter_y[6] >> 1);
auto f7 = hn::Set(tag16, filter_y[7] >> 1);
auto round_offset = hn::Set(tag16, 1 << (FILTER_BITS - 2));
if (w == 4) {
for (int y = 0; y < h; y += 4) {
auto s0 = LoadUnaligned4x4(tag16, src + (y + 0) * src_stride, src_stride);
auto s1 = LoadUnaligned4x4(tag16, src + (y + 1) * src_stride, src_stride);
auto s2 = LoadUnaligned4x4(tag16, src + (y + 2) * src_stride, src_stride);
auto s3 = LoadUnaligned4x4(tag16, src + (y + 3) * src_stride, src_stride);
auto s4 = LoadUnaligned4x4(tag16, src + (y + 4) * src_stride, src_stride);
auto s5 = LoadUnaligned4x4(tag16, src + (y + 5) * src_stride, src_stride);
auto s6 = LoadUnaligned4x4(tag16, src + (y + 6) * src_stride, src_stride);
auto s7 = LoadUnaligned4x4(tag16, src + (y + 7) * src_stride, src_stride);
auto res = hn::ShiftRight<FILTER_BITS - 1>(
s0 * f0 + s1 * f1 + s2 * f2 + s3 * f3 + s4 * f4 + s5 * f5 + s6 * f6 +
s7 * f7 + round_offset);
StoreUnaligned4x4(tag16, dst + y * dst_stride, dst_stride, res);
}
} else if (w == 8) {
auto s0 = LoadUnaligned2x8(tag16, src + 0 * src_stride, src_stride);
auto s1 = LoadUnaligned2x8(tag16, src + 1 * src_stride, src_stride);
auto s2 = LoadUnaligned2x8(tag16, src + 2 * src_stride, src_stride);
auto s3 = LoadUnaligned2x8(tag16, src + 3 * src_stride, src_stride);
auto s4 = LoadUnaligned2x8(tag16, src + 4 * src_stride, src_stride);
auto s5 = LoadUnaligned2x8(tag16, src + 5 * src_stride, src_stride);
for (int y = 0; y < h; y += 2) {
auto s6 = LoadUnaligned2x8(tag16, src + (y + 6) * src_stride, src_stride);
auto s7 = LoadUnaligned2x8(tag16, src + (y + 7) * src_stride, src_stride);
auto res = hn::ShiftRight<FILTER_BITS - 1>(
s0 * f0 + s1 * f1 + s2 * f2 + s3 * f3 + s4 * f4 + s5 * f5 + s6 * f6 +
s7 * f7 + round_offset);
StoreUnaligned2x8(tag16, dst + y * dst_stride, dst_stride, res);
s0 = s2;
s1 = s3;
s2 = s4;
s3 = s5;
s4 = s6;
s5 = s7;
}
} else if (w == 16) {
constexpr hn::Half<decltype(tag16)> half_tag16;
auto s0 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 0 * src_stride));
auto s1 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 1 * src_stride));
auto s2 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 2 * src_stride));
auto s3 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 3 * src_stride));
auto s4 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 4 * src_stride));
auto s5 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 5 * src_stride));
auto s6 = hn::PromoteTo(tag16, hn::LoadU(pixel_tag, src + 6 * src_stride));
for (int y = 0; y < h; ++y) {
auto s7 = hn::PromoteTo(tag16,
hn::LoadU(pixel_tag, src + (y + 7) * src_stride));
auto res = hn::ShiftRight<FILTER_BITS - 1>(
s0 * f0 + s1 * f1 + s2 * f2 + s3 * f3 + s4 * f4 + s5 * f5 + s6 * f6 +
s7 * f7 + round_offset);
hn::StoreU(hn::ReorderDemote2To(pixel_tag, hn::LowerHalf(half_tag16, res),
hn::UpperHalf(half_tag16, res)),
pixel_tag, dst + y * dst_stride);
s0 = s1;
s1 = s2;
s2 = s3;
s3 = s4;
s4 = s5;
s5 = s6;
s6 = s7;
}
} else {
hn::ScalableTag<int16_t> mul_tag;
hn::Rebind<uint8_t, decltype(mul_tag)> p_tag;
auto f0_s = hn::Set(mul_tag, filter_y[0] >> 1);
auto f1_s = hn::Set(mul_tag, filter_y[1] >> 1);
auto f2_s = hn::Set(mul_tag, filter_y[2] >> 1);
auto f3_s = hn::Set(mul_tag, filter_y[3] >> 1);
auto f4_s = hn::Set(mul_tag, filter_y[4] >> 1);
auto f5_s = hn::Set(mul_tag, filter_y[5] >> 1);
auto f6_s = hn::Set(mul_tag, filter_y[6] >> 1);
auto f7_s = hn::Set(mul_tag, filter_y[7] >> 1);
auto round_offset_s = hn::Set(mul_tag, 1 << (FILTER_BITS - 2));
auto vw = hn::Lanes(mul_tag);
for (int x = 0; x < w; x += vw) {
auto s0 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 0 * src_stride));
auto s1 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 1 * src_stride));
auto s2 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 2 * src_stride));
auto s3 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 3 * src_stride));
auto s4 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 4 * src_stride));
auto s5 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 5 * src_stride));
auto s6 =
hn::PromoteTo(mul_tag, hn::LoadU(p_tag, src + x + 6 * src_stride));
for (int y = 0; y < h; ++y) {
auto s7 = hn::PromoteTo(
mul_tag, hn::LoadU(p_tag, src + x + (y + 7) * src_stride));
auto sum = s0 * f0_s + s1 * f1_s + s2 * f2_s + s3 * f3_s + s4 * f4_s +
s5 * f5_s + s6 * f6_s + s7 * f7_s;
auto res = hn::ShiftRight<FILTER_BITS - 1>(sum + round_offset_s);
auto res_demoted = hn::DemoteTo(p_tag, res);
if (x + static_cast<int>(vw) > w) {
hn::StoreN(res_demoted, p_tag, dst + x + y * dst_stride, w - x);
} else {
hn::StoreU(res_demoted, p_tag, dst + x + y * dst_stride);
}
s0 = s1;
s1 = s2;
s2 = s3;
s3 = s4;
s4 = s5;
s5 = s6;
s6 = s7;
}
}
}
}
HWY_MAYBE_UNUSED void Convolve8Vert(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h) {
assert((intptr_t)dst % 4 == 0);
assert(dst_stride % 4 == 0);
(void)x_step_q4;
(void)filter_x;
(void)y_step_q4;
src -= src_stride * ((SUBPEL_TAPS / 2) - 1);
int filter_taps = get_filter_taps_convolve8(filter_y);
if (filter_taps == 2) {
ConvolveVert2Tap(src + src_stride * 3, src_stride, dst, dst_stride,
filter_y, w, h);
} else if (filter_taps == 4) {
ConvolveVert4Tap(src + src_stride * 2, src_stride, dst, dst_stride,
filter_y, w, h);
} else {
// filter_taps = 8
ConvolveVert8Tap(src, src_stride, dst, dst_stride, filter_y, w, h);
}
}
HWY_MAYBE_UNUSED void Convolve8Horiz(const uint8_t *src, ptrdiff_t src_stride,
uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *filter_x, int x_step_q4,
const int16_t *filter_y, int y_step_q4,
int w, int h) {
#if HAVE_SSSE3
// TODO: jianj - 16x32 block is still fastest with handwritten avx2 which
// uses ssse3 implementation. Further optimize for this case.
if (w == 16 && h == 32) {
aom_convolve8_horiz_ssse3(src, src_stride, dst, dst_stride, filter_x,
x_step_q4, filter_y, y_step_q4, w, h);
return;
}
#endif
assert((intptr_t)dst % 4 == 0);
assert(dst_stride % 4 == 0);
(void)x_step_q4;
(void)filter_y;
(void)y_step_q4;
src -= ((SUBPEL_TAPS / 2) - 1);
int filter_taps = get_filter_taps_convolve8(filter_x);
if (filter_taps == 2) {
ConvolveHoriz2Tap(src + 3, src_stride, dst, dst_stride, filter_x, w, h);
} else if (filter_taps == 4) {
ConvolveHoriz4Tap(src + 2, src_stride, dst, dst_stride, filter_x, w, h);
} else {
// filter_taps = 8
ConvolveHoriz8Tap(src, src_stride, dst, dst_stride, filter_x, w, h);
}
}
} // namespace HWY_NAMESPACE
} // namespace
#define CONVOLVE8HORIZ(suffix) \
extern "C" void aom_convolve8_horiz_##suffix( \
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \
const int16_t *filter_y, int y_step_q4, int w, int h); \
HWY_ATTR void aom_convolve8_horiz_##suffix( \
const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \
ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, \
const int16_t *filter_y, int y_step_q4, int w, int h) { \
HWY_NAMESPACE::Convolve8Horiz(src, src_stride, dst, dst_stride, filter_x, \
x_step_q4, filter_y, y_step_q4, w, h); \
}
HWY_AFTER_NAMESPACE();
#endif // AOM_AOM_DSP_CONVOLVE_HWY_H_