blob: 6ce320ee2867936dba8261dad5a8e4342aa3d5bc [file]
/*
* Copyright (c) 2026, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_AOM_DSP_VARIANCE_HWY_H_
#define AOM_AOM_DSP_VARIANCE_HWY_H_
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/blend.h"
#include "aom_dsp/reduce_sum_hwy.h"
#include "third_party/highway/hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace {
namespace HWY_NAMESPACE {
namespace hn = hwy::HWY_NAMESPACE;
template <class D, class VHalf>
HWY_ATTR HWY_INLINE hn::VFromD<D> Combine2Rows(D d, VHalf r0, VHalf r1) {
HWY_IF_CONSTEXPR(hn::MaxLanes(d) * sizeof(hn::TFromD<D>) >= 32) {
return hn::ConcatLowerLower(d, hn::ResizeBitCast(d, r1),
hn::ResizeBitCast(d, r0));
}
return hn::Combine(d, r1, r0);
}
template <class D, class VQuarter>
HWY_ATTR HWY_INLINE hn::VFromD<D> Combine4Rows(D d, VQuarter r0, VQuarter r1,
VQuarter r2, VQuarter r3) {
HWY_IF_CONSTEXPR(hn::MaxLanes(d) * sizeof(hn::TFromD<D>) == 64) {
auto v = hn::ResizeBitCast(d, r0);
v = hn::InsertBlock<1>(v, r1);
v = hn::InsertBlock<2>(v, r2);
return hn::InsertBlock<3>(v, r3);
}
constexpr hn::Half<D> d_half;
auto r01 = Combine2Rows(d_half, r0, r1);
auto r23 = Combine2Rows(d_half, r2, r3);
return Combine2Rows(d, r01, r23);
}
template <int RowsPerVec, class D>
HWY_ATTR HWY_INLINE hn::VFromD<D> LoadRowsU(D full_tag,
const hn::TFromD<D> *ptr,
ptrdiff_t stride) {
HWY_IF_CONSTEXPR(RowsPerVec == 4) {
constexpr hn::Half<hn::Half<D>> row_tag;
const hn::TFromD<D> *ptr2 = ptr + stride * 2;
return Combine4Rows(
full_tag, hn::LoadU(row_tag, ptr), hn::LoadU(row_tag, ptr + stride),
hn::LoadU(row_tag, ptr2), hn::LoadU(row_tag, ptr2 + stride));
}
HWY_IF_CONSTEXPR(RowsPerVec == 2) {
constexpr hn::Half<D> row_tag;
return Combine2Rows(full_tag, hn::LoadU(row_tag, ptr),
hn::LoadU(row_tag, ptr + stride));
}
return hn::LoadU(full_tag, ptr);
}
template <class DU8, class DI32, class DI16>
HWY_ATTR HWY_INLINE void VarianceKernel(DU8 du8, DI32 di32, DI16 di16,
hn::VFromD<DU8> src,
hn::VFromD<DU8> ref,
hn::VFromD<DI32> *sse,
hn::VFromD<DI16> *sum) {
constexpr hn::Repartition<int8_t, DU8> di8;
const auto adj_sub =
hn::BitCast(di8, hn::Set(di16, static_cast<int16_t>(0xff01)));
const auto s_lo = hn::InterleaveLower(du8, src, ref);
const auto s_hi = hn::InterleaveUpper(du8, src, ref);
const auto diff_lo = hn::SatWidenMulPairwiseAdd(di16, s_lo, adj_sub);
const auto diff_hi = hn::SatWidenMulPairwiseAdd(di16, s_hi, adj_sub);
*sum = hn::Add(*sum, hn::Add(diff_lo, diff_hi));
const auto madd_lo = hn::WidenMulPairwiseAdd(di32, diff_lo, diff_lo);
const auto madd_hi = hn::WidenMulPairwiseAdd(di32, diff_hi, diff_hi);
*sse = hn::Add(*sse, hn::Add(madd_lo, madd_hi));
}
template <class DI32>
HWY_ATTR HWY_INLINE int VarianceFinalFrom32BitSum(DI32 di32,
hn::VFromD<DI32> vsse,
hn::VFromD<DI32> vsum32,
unsigned int *const sse) {
constexpr hn::BlockDFromD<DI32> d128;
const auto sse128 = BlockReduceSum(di32, vsse);
const auto sum128 = BlockReduceSum(di32, vsum32);
const auto pair0 = hn::InterleaveLower(d128, sse128, sum128);
const auto pair1 = hn::InterleaveUpper(d128, sse128, sum128);
const auto sum_pair = hn::Add(pair0, pair1);
constexpr hn::FixedTag<int32_t, 2> d64;
const auto res =
hn::Add(hn::LowerHalf(d64, sum_pair), hn::UpperHalf(d64, sum_pair));
*sse = static_cast<unsigned int>(hn::GetLane(res));
return hn::ExtractLane(res, 1);
}
template <class DI32, class DI16>
HWY_ATTR HWY_INLINE int VarianceFinalFrom16BitSum(DI32 di32, DI16 di16,
hn::VFromD<DI32> vsse,
hn::VFromD<DI16> vsum,
unsigned int *const sse) {
const auto one = hn::Set(di16, int16_t{ 1 });
const auto vsum32 = hn::WidenMulPairwiseAdd(di32, vsum, one);
return VarianceFinalFrom32BitSum(di32, vsse, vsum32, sse);
}
template <int BlockWidth>
HWY_ATTR HWY_INLINE void VarianceBlock(
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride,
int h, hn::Vec<hn::ScalableTag<int32_t>> *vsse,
hn::Vec<hn::ScalableTag<int16_t>> *vsum) {
constexpr hn::ScalableTag<uint8_t> du8;
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::ScalableTag<int32_t> di32;
constexpr int vw = hn::MaxLanes(du8);
constexpr int kRowsPerVec = (BlockWidth < vw) ? (vw / BlockWidth) : 1;
constexpr int kRowSpan = (BlockWidth < vw) ? vw : BlockWidth;
assert(h % kRowsPerVec == 0);
*vsum = hn::Zero(di16);
for (int i = 0; i < h; i += kRowsPerVec) {
for (int j = 0; j < kRowSpan; j += vw) {
const auto s = LoadRowsU<kRowsPerVec>(du8, src + j, src_stride);
const auto r = LoadRowsU<kRowsPerVec>(du8, ref + j, ref_stride);
VarianceKernel(du8, di32, di16, s, r, vsse, vsum);
}
src += kRowsPerVec * src_stride;
ref += kRowsPerVec * ref_stride;
}
}
template <int BlockWidth, int BlockHeight>
HWY_ATTR HWY_INLINE void GetBlockVar(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
unsigned int *sse, int *sum) {
constexpr hn::ScalableTag<uint8_t> du8;
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::ScalableTag<int32_t> di32;
constexpr int vw = hn::MaxLanes(du8);
constexpr int kRowsPerVec = (BlockWidth < vw) ? (vw / BlockWidth) : 1;
// Each int16_t lane in vsum accumulates 2 pixel diffs per iteration of vw
// bytes. With 128 diffs per lane (128 * 255 = 32640 <= 32767), we can safely
// accumulate up to 64 * vw pixels before widening vsum to 32-bit.
constexpr int kMaxPixelsPerChunk = 64 * vw;
constexpr int kChunkHeight = (BlockWidth * BlockHeight <= kMaxPixelsPerChunk)
? BlockHeight
: (kMaxPixelsPerChunk / BlockWidth);
static_assert(BlockHeight % kRowsPerVec == 0,
"BlockHeight must be a multiple of kRowsPerVec.");
static_assert(BlockHeight % kChunkHeight == 0,
"BlockHeight must be a multiple of kChunkHeight.");
static_assert(kChunkHeight % kRowsPerVec == 0,
"kChunkHeight must be a multiple of kRowsPerVec.");
auto vsse = hn::Zero(di32);
HWY_IF_CONSTEXPR(kChunkHeight == BlockHeight) {
hn::Vec<decltype(di16)> vsum;
VarianceBlock<BlockWidth>(src, src_stride, ref, ref_stride, BlockHeight,
&vsse, &vsum);
*sum = VarianceFinalFrom16BitSum(di32, di16, vsse, vsum, sse);
}
HWY_IF_CONSTEXPR(kChunkHeight != BlockHeight) {
auto vsum32 = hn::Zero(di32);
const auto one = hn::Set(di16, int16_t{ 1 });
for (int i = 0; i < BlockHeight / kChunkHeight; ++i) {
hn::Vec<decltype(di16)> vsum16;
VarianceBlock<BlockWidth>(src, src_stride, ref, ref_stride, kChunkHeight,
&vsse, &vsum16);
vsum32 = hn::Add(vsum32, hn::WidenMulPairwiseAdd(di32, vsum16, one));
src += kChunkHeight * src_stride;
ref += kChunkHeight * ref_stride;
}
*sum = VarianceFinalFrom32BitSum(di32, vsse, vsum32, sse);
}
}
template <int BlockWidth, int BlockHeight, int Bits>
HWY_ATTR HWY_INLINE unsigned int VarianceWxH(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
unsigned int *sse) {
int sum = 0;
GetBlockVar<BlockWidth, BlockHeight>(src, src_stride, ref, ref_stride, sse,
&sum);
return *sse -
static_cast<unsigned int>((static_cast<int64_t>(sum) * sum) >> Bits);
}
HWY_ATTR HWY_INLINE void GetBlkSseSum(const int16_t *data, int stride, int bw,
int bh, int *x_sum, int64_t *x2_sum) {
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::ScalableTag<int32_t> di32;
constexpr hn::ScalableTag<uint32_t> du32;
constexpr hn::ScalableTag<uint64_t> du64;
constexpr int kLanes = hn::MaxLanes(di16);
if (bw <= 0 || bh <= 0) {
*x_sum = 0;
*x2_sum = 0;
return;
}
if (bw < kLanes || bw % kLanes != 0) {
aom_get_blk_sse_sum_avx2(data, stride, bw, bh, x_sum, x2_sum);
return;
}
const auto one = hn::Set(di16, int16_t{ 1 });
auto sum_32 = hn::Zero(di32);
auto sse_64 = hn::Zero(du64);
for (int i = 0; i < bh; ++i) {
for (int j = 0; j < bw; j += kLanes) {
const auto d = hn::LoadU(di16, data + j);
sum_32 = hn::Add(sum_32, hn::WidenMulPairwiseAdd(di32, d, one));
const auto sq = hn::BitCast(du32, hn::WidenMulPairwiseAdd(di32, d, d));
const auto sq_lo = hn::PromoteLowerTo(du64, sq);
const auto sq_hi = hn::PromoteUpperTo(du64, sq);
sse_64 = hn::Add(sse_64, hn::Add(sq_lo, sq_hi));
}
data += stride;
}
*x_sum = static_cast<int>(hn::ReduceSum(di32, sum_32));
*x2_sum = static_cast<int64_t>(hn::ReduceSum(du64, sse_64));
}
HWY_ATTR HWY_INLINE void LoadRowHorizFilterU8(
const uint8_t *src_row, int x, int bw,
hn::Vec<hn::ScalableTag<int16_t>> *u16_curr_out,
hn::Vec<hn::ScalableTag<int16_t>> *u16_H_out) {
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::Rebind<uint8_t, decltype(di16)> du8_half;
constexpr int kCols = hn::MaxLanes(du8_half);
static_assert(kCols == 16 || kCols == 32, "kCols must be 16 or 32.");
alignas(64) static constexpr int16_t kLeftIdx[32] = {
0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30
};
alignas(64) static constexpr int16_t kRightIdx32[32] = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 31
};
alignas(32) static constexpr int16_t kRightIdx16[16] = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15
};
const auto u16_curr = hn::PromoteTo(di16, hn::LoadU(du8_half, src_row + x));
hn::Vec<decltype(di16)> u16_left;
hn::Vec<decltype(di16)> u16_right;
if (x == 0) {
u16_left =
hn::TableLookupLanes(u16_curr, hn::SetTableIndices(di16, kLeftIdx));
} else {
u16_left = hn::PromoteTo(di16, hn::LoadU(du8_half, src_row + x - 1));
}
if (x + kCols < bw) {
u16_right = hn::PromoteTo(di16, hn::LoadU(du8_half, src_row + x + 1));
} else {
const int16_t *right_idx = (kCols == 32) ? kRightIdx32 : kRightIdx16;
u16_right =
hn::TableLookupLanes(u16_curr, hn::SetTableIndices(di16, right_idx));
}
*u16_curr_out = u16_curr;
*u16_H_out =
hn::Add(hn::Add(u16_left, u16_right), hn::ShiftLeft<1>(u16_curr));
}
HWY_ATTR HWY_INLINE int64_t CalcVarianceStat(const uint8_t *src, int stride,
int bw, int bh) {
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::ScalableTag<int32_t> di32;
constexpr int kCols = hn::MaxLanes(di16);
if (bw <= 0 || bh <= 0) return 0;
if (bw == 4 || bw == 8 || bw == 16) {
return aom_calc_variance_stat_avx2(src, stride, bw, bh);
}
if (bw % kCols != 0) {
return aom_calc_variance_stat_c(src, stride, bw, bh);
}
auto acc_var = hn::Zero(di32);
for (int x = 0; x < bw; x += kCols) {
hn::Vec<decltype(di16)> u16_curr, H_curr;
LoadRowHorizFilterU8(src, x, bw, &u16_curr, &H_curr);
auto H_top = H_curr;
for (int y = 0; y < bh - 1; ++y) {
hn::Vec<decltype(di16)> u16_next, H_bot;
LoadRowHorizFilterU8(src + (y + 1) * stride, x, bw, &u16_next, &H_bot);
const auto u16_V =
hn::Add(hn::Add(H_top, H_bot), hn::ShiftLeft<1>(H_curr));
const auto u16_sum = hn::ShiftRight<4>(u16_V);
const auto diff = hn::Sub(u16_curr, u16_sum);
acc_var = hn::Add(acc_var, hn::WidenMulPairwiseAdd(di32, diff, diff));
H_top = H_curr;
H_curr = H_bot;
u16_curr = u16_next;
}
{
const auto u16_V =
hn::Add(hn::Add(H_top, H_curr), hn::ShiftLeft<1>(H_curr));
const auto u16_sum = hn::ShiftRight<4>(u16_V);
const auto diff = hn::Sub(u16_curr, u16_sum);
acc_var = hn::Add(acc_var, hn::WidenMulPairwiseAdd(di32, diff, diff));
}
}
return static_cast<int64_t>(hn::ReduceSum(di32, acc_var)) << 4;
}
#if CONFIG_AV1_HIGHBITDEPTH
HWY_ATTR HWY_INLINE void HighbdLoadRowHorizFilterU16(
const uint16_t *src_row, int x, int bw,
hn::Vec<hn::ScalableTag<int16_t>> *v_curr_out,
hn::Vec<hn::ScalableTag<uint16_t>> *u16_H_out) {
constexpr hn::ScalableTag<uint16_t> du16;
constexpr hn::ScalableTag<int16_t> di16;
constexpr int kCols = hn::MaxLanes(du16);
static_assert(kCols == 16 || kCols == 32, "kCols must be 16 or 32.");
alignas(64) static constexpr uint16_t kLeftIdx[32] = {
0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30
};
alignas(64) static constexpr uint16_t kRightIdx32[32] = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 31
};
alignas(32) static constexpr uint16_t kRightIdx16[16] = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15
};
const auto v_curr = hn::LoadU(du16, src_row + x);
// Input pixels must be <= 4095 (12-bit) so that the 3x3 Gaussian filter sum
// u16_V = H_top + 2 * H_curr + H_bot <= 16 * 4095 = 65520 fits in uint16_t.
assert(hn::AllTrue(du16, hn::Le(v_curr, hn::Set(du16, uint16_t{ 4095 }))));
hn::Vec<decltype(du16)> v_left;
hn::Vec<decltype(du16)> v_right;
if (x == 0) {
v_left = hn::TableLookupLanes(v_curr, hn::SetTableIndices(du16, kLeftIdx));
} else {
v_left = hn::LoadU(du16, src_row + x - 1);
}
if (x + kCols < bw) {
v_right = hn::LoadU(du16, src_row + x + 1);
} else {
const uint16_t *right_idx = (kCols == 32) ? kRightIdx32 : kRightIdx16;
v_right =
hn::TableLookupLanes(v_curr, hn::SetTableIndices(du16, right_idx));
}
*v_curr_out = hn::BitCast(di16, v_curr);
*u16_H_out = hn::Add(hn::Add(v_left, v_right), hn::ShiftLeft<1>(v_curr));
}
// Assumes 8/10/12-bit input pixels (src[i] <= 4095) so that the 16-weight 3x3
// Gaussian filter sum (<= 65520) does not overflow uint16_t.
HWY_ATTR HWY_INLINE int64_t HighbdCalcVarianceStat(const uint16_t *src,
int stride, int bw, int bh) {
constexpr hn::ScalableTag<uint16_t> du16;
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::ScalableTag<int32_t> di32;
constexpr hn::ScalableTag<int64_t> di64;
constexpr int kCols = hn::MaxLanes(du16);
if (bw <= 0 || bh <= 0) return 0;
if (bw == 4 || bw == 8 || bw == 16) {
return aom_highbd_calc_variance_stat_avx2(src, stride, bw, bh);
}
if (bw % kCols != 0) {
return aom_highbd_calc_variance_stat_c(src, stride, bw, bh);
}
auto acc_var_64 = hn::Zero(di64);
for (int x = 0; x < bw; x += kCols) {
hn::Vec<decltype(di16)> v_curr;
hn::Vec<decltype(du16)> H_curr;
HighbdLoadRowHorizFilterU16(src, x, bw, &v_curr, &H_curr);
auto H_top = H_curr;
for (int y = 0; y < bh - 1; ++y) {
hn::Vec<decltype(di16)> v_next;
hn::Vec<decltype(du16)> H_bot;
HighbdLoadRowHorizFilterU16(src + (y + 1) * stride, x, bw, &v_next,
&H_bot);
const auto u16_V =
hn::Add(hn::Add(H_top, H_bot), hn::ShiftLeft<1>(H_curr));
const auto u16_sum = hn::BitCast(di16, hn::ShiftRight<4>(u16_V));
const auto diff = hn::Sub(v_curr, u16_sum);
const auto diff_sq = hn::WidenMulPairwiseAdd(di32, diff, diff);
acc_var_64 =
hn::Add(acc_var_64, hn::Add(hn::PromoteLowerTo(di64, diff_sq),
hn::PromoteUpperTo(di64, diff_sq)));
H_top = H_curr;
H_curr = H_bot;
v_curr = v_next;
}
{
const auto u16_V =
hn::Add(hn::Add(H_top, H_curr), hn::ShiftLeft<1>(H_curr));
const auto u16_sum = hn::BitCast(di16, hn::ShiftRight<4>(u16_V));
const auto diff = hn::Sub(v_curr, u16_sum);
const auto diff_sq = hn::WidenMulPairwiseAdd(di32, diff, diff);
acc_var_64 =
hn::Add(acc_var_64, hn::Add(hn::PromoteLowerTo(di64, diff_sq),
hn::PromoteUpperTo(di64, diff_sq)));
}
}
return static_cast<int64_t>(hn::ReduceSum(di64, acc_var_64)) << 4;
}
#endif // CONFIG_AV1_HIGHBITDEPTH
// -----------------------------------------------------------------------------
// Subpixel Variance & Subpixel Avg Variance
// -----------------------------------------------------------------------------
template <class DI8>
HWY_ATTR HWY_INLINE hn::VFromD<DI8> BilinearFilter(DI8 di8, int offset) {
assert(offset >= 0 && offset < 8);
constexpr hn::Repartition<int16_t, DI8> di16;
const int f0 = 16 - 2 * offset;
const int f1 = 2 * offset;
return hn::BitCast(di8, hn::Set(di16, static_cast<int16_t>(f0 | (f1 << 8))));
}
template <class DU8, class DI16>
HWY_ATTR HWY_INLINE void FilterPair(
DU8 du8, DI16 di16, hn::VFromD<DU8> s0, hn::VFromD<DU8> s1,
hn::VFromD<hn::Repartition<int8_t, DU8>> filt, hn::VFromD<DI16> *out_lo,
hn::VFromD<DI16> *out_hi) {
const auto pw8 = hn::Set(di16, int16_t{ 8 });
const auto lo =
hn::SatWidenMulPairwiseAdd(di16, hn::InterleaveLower(du8, s0, s1), filt);
const auto hi =
hn::SatWidenMulPairwiseAdd(di16, hn::InterleaveUpper(du8, s0, s1), filt);
*out_lo = hn::ShiftRight<4>(hn::Add(lo, pw8));
*out_hi = hn::ShiftRight<4>(hn::Add(hi, pw8));
}
template <class DU8>
HWY_ATTR HWY_INLINE hn::VFromD<DU8> FilterPairPack(
DU8 du8, hn::VFromD<DU8> s0, hn::VFromD<DU8> s1,
hn::VFromD<hn::Repartition<int8_t, DU8>> filt) {
constexpr hn::Repartition<int16_t, DU8> di16;
hn::VFromD<decltype(di16)> lo, hi;
FilterPair(du8, di16, s0, s1, filt, &lo, &hi);
return hn::ReorderDemote2To(du8, lo, hi);
}
template <class DU8, class DI16, class DI32>
HWY_ATTR HWY_INLINE void AccumDiff(DU8 du8, DI16 di16, DI32 di32,
hn::VFromD<DI16> exp_src_lo,
hn::VFromD<DI16> exp_src_hi,
hn::VFromD<DU8> dst_reg,
hn::VFromD<DI32> *sse_reg,
hn::VFromD<DI16> *sum_reg) {
const auto zero = hn::Zero(du8);
const auto exp_dst_lo =
hn::BitCast(di16, hn::InterleaveLower(du8, dst_reg, zero));
const auto exp_dst_hi =
hn::BitCast(di16, hn::InterleaveUpper(du8, dst_reg, zero));
const auto d_lo = hn::Sub(exp_src_lo, exp_dst_lo);
const auto d_hi = hn::Sub(exp_src_hi, exp_dst_hi);
*sum_reg = hn::Add(*sum_reg, hn::Add(d_lo, d_hi));
const auto madd_lo = hn::WidenMulPairwiseAdd(di32, d_lo, d_lo);
const auto madd_hi = hn::WidenMulPairwiseAdd(di32, d_hi, d_hi);
*sse_reg = hn::Add(*sse_reg, hn::Add(madd_lo, madd_hi));
}
template <int Width, class D>
HWY_ATTR HWY_INLINE hn::VFromD<D> SubPelLoad(D d, const uint8_t *ptr,
ptrdiff_t half_stride) {
HWY_IF_CONSTEXPR(Width == 32 && hn::MaxLanes(d) == 64) {
constexpr hn::Half<D> d_half;
return Combine2Rows(d, hn::LoadU(d_half, ptr),
hn::LoadU(d_half, ptr + half_stride));
}
return hn::LoadU(d, ptr);
}
template <int Width, bool HasSec, class LoadRowFn>
HWY_ATTR HWY_INLINE void SubPelVertLoop(
const uint8_t *s_ptr, int src_stride, int y_offset, const uint8_t *d_ptr,
int dst_stride, ptrdiff_t half_dst_stride, const uint8_t *sec_ptr,
int sec_stride, ptrdiff_t half_sec_stride, int loop_h, LoadRowFn load_row,
hn::Vec<hn::ScalableTag<int32_t>> *sse_reg,
hn::Vec<hn::ScalableTag<int16_t>> *sum_reg) {
constexpr hn::ScalableTag<uint8_t> du8;
constexpr hn::Repartition<int8_t, decltype(du8)> di8;
constexpr hn::Repartition<int16_t, decltype(du8)> di16;
constexpr hn::Repartition<int32_t, decltype(du8)> di32;
if (y_offset == 0) {
for (int i = 0; i < loop_h; ++i) {
auto s = load_row(s_ptr);
HWY_IF_CONSTEXPR(HasSec) {
s = hn::AverageRound(s,
SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
sec_ptr += sec_stride;
}
const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
VarianceKernel(du8, di32, di16, s, d, sse_reg, sum_reg);
s_ptr += src_stride;
d_ptr += dst_stride;
}
} else if (y_offset == 4) {
auto s_prev = load_row(s_ptr);
for (int i = 0; i < loop_h; ++i) {
s_ptr += src_stride;
const auto s_curr = load_row(s_ptr);
auto s_avg = hn::AverageRound(s_prev, s_curr);
HWY_IF_CONSTEXPR(HasSec) {
s_avg = hn::AverageRound(
s_avg, SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
sec_ptr += sec_stride;
}
const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
VarianceKernel(du8, di32, di16, s_avg, d, sse_reg, sum_reg);
s_prev = s_curr;
d_ptr += dst_stride;
}
} else {
const auto yfilter = BilinearFilter(di8, y_offset);
auto s_prev = load_row(s_ptr);
for (int i = 0; i < loop_h; ++i) {
s_ptr += src_stride;
const auto s_curr = load_row(s_ptr);
const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
HWY_IF_CONSTEXPR(HasSec) {
const auto s_avg =
hn::AverageRound(FilterPairPack(du8, s_prev, s_curr, yfilter),
SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
VarianceKernel(du8, di32, di16, s_avg, d, sse_reg, sum_reg);
sec_ptr += sec_stride;
}
HWY_IF_CONSTEXPR(!HasSec) {
hn::VFromD<decltype(di16)> exp_lo, exp_hi;
FilterPair(du8, di16, s_prev, s_curr, yfilter, &exp_lo, &exp_hi);
AccumDiff(du8, di16, di32, exp_lo, exp_hi, d, sse_reg, sum_reg);
}
s_prev = s_curr;
d_ptr += dst_stride;
}
}
}
template <int Width, bool HasSec>
HWY_ATTR HWY_INLINE int SubPixelVarianceWxHImp(
const uint8_t *src, int src_stride, int x_offset, int y_offset,
const uint8_t *dst, int dst_stride, const uint8_t *sec, int sec_stride,
int height, unsigned int *sse) {
constexpr hn::ScalableTag<uint8_t> du8;
constexpr hn::Repartition<int8_t, decltype(du8)> di8;
constexpr hn::Repartition<int16_t, decltype(du8)> di16;
constexpr hn::Repartition<int32_t, decltype(du8)> di32;
constexpr int vw = hn::MaxLanes(du8);
constexpr int kRowsPerIter = (Width == 32 && vw == 64) ? 2 : 1;
constexpr int kColStep = (Width == 32 && vw == 64) ? 32 : vw;
static_assert(Width == 32 || Width % kColStep == 0,
"Width must be 32 or a multiple of kColStep.");
const int loop_h = height / kRowsPerIter;
// Each int16_t lane in sum_reg accumulates 2 pixel diffs per iteration, so
// loop_h must be <= 64 (128 * 255 = 32640 <= 32767) to avoid overflow.
assert(height > 0 && height % kRowsPerIter == 0 && loop_h <= 64);
const ptrdiff_t half_src_stride = static_cast<ptrdiff_t>(loop_h) * src_stride;
const ptrdiff_t half_dst_stride = static_cast<ptrdiff_t>(loop_h) * dst_stride;
const ptrdiff_t half_sec_stride = static_cast<ptrdiff_t>(loop_h) * sec_stride;
const auto one = hn::Set(di16, int16_t{ 1 });
auto sum32_reg = hn::Zero(di32);
auto sse_reg = hn::Zero(di32);
for (int col = 0; col < Width; col += kColStep) {
auto sum_reg = hn::Zero(di16);
const uint8_t *s_ptr = src + col;
const uint8_t *d_ptr = dst + col;
const uint8_t *sec_ptr = HasSec ? (sec + col) : nullptr;
if (x_offset == 0) {
SubPelVertLoop<Width, HasSec>(
s_ptr, src_stride, y_offset, d_ptr, dst_stride, half_dst_stride,
sec_ptr, sec_stride, half_sec_stride, loop_h,
[=](const uint8_t *p)
HWY_ATTR { return SubPelLoad<Width>(du8, p, half_src_stride); },
&sse_reg, &sum_reg);
} else if (x_offset == 4) {
SubPelVertLoop<Width, HasSec>(
s_ptr, src_stride, y_offset, d_ptr, dst_stride, half_dst_stride,
sec_ptr, sec_stride, half_sec_stride, loop_h,
[=](const uint8_t *p) HWY_ATTR {
return hn::AverageRound(
SubPelLoad<Width>(du8, p, half_src_stride),
SubPelLoad<Width>(du8, p + 1, half_src_stride));
},
&sse_reg, &sum_reg);
} else {
const auto xfilter = BilinearFilter(di8, x_offset);
if (y_offset == 0) {
for (int i = 0; i < loop_h; ++i) {
const auto s0 = SubPelLoad<Width>(du8, s_ptr, half_src_stride);
const auto s1 = SubPelLoad<Width>(du8, s_ptr + 1, half_src_stride);
const auto d = SubPelLoad<Width>(du8, d_ptr, half_dst_stride);
HWY_IF_CONSTEXPR(HasSec) {
const auto s = hn::AverageRound(
FilterPairPack(du8, s0, s1, xfilter),
SubPelLoad<Width>(du8, sec_ptr, half_sec_stride));
VarianceKernel(du8, di32, di16, s, d, &sse_reg, &sum_reg);
sec_ptr += sec_stride;
}
HWY_IF_CONSTEXPR(!HasSec) {
hn::VFromD<decltype(di16)> exp_lo, exp_hi;
FilterPair(du8, di16, s0, s1, xfilter, &exp_lo, &exp_hi);
AccumDiff(du8, di16, di32, exp_lo, exp_hi, d, &sse_reg, &sum_reg);
}
s_ptr += src_stride;
d_ptr += dst_stride;
}
} else {
SubPelVertLoop<Width, HasSec>(
s_ptr, src_stride, y_offset, d_ptr, dst_stride, half_dst_stride,
sec_ptr, sec_stride, half_sec_stride, loop_h,
[=](const uint8_t *p) HWY_ATTR {
return FilterPairPack(
du8, SubPelLoad<Width>(du8, p, half_src_stride),
SubPelLoad<Width>(du8, p + 1, half_src_stride), xfilter);
},
&sse_reg, &sum_reg);
}
}
sum32_reg = hn::Add(sum32_reg, hn::WidenMulPairwiseAdd(di32, sum_reg, one));
}
return VarianceFinalFrom32BitSum(di32, sse_reg, sum32_reg, sse);
}
template <int BlockWidth, int BlockHeight, int ChunkHeight, int WLog2,
int HLog2, bool HasSec>
HWY_ATTR HWY_INLINE unsigned int SubPixelVarianceWxH(
const uint8_t *src, int src_stride, int x_offset, int y_offset,
const uint8_t *dst, int dst_stride, unsigned int *sse_ptr,
const uint8_t *sec) {
constexpr int kChunkWidth = (BlockWidth == 32) ? 32 : 64;
static_assert(BlockWidth == 32 || BlockWidth % 64 == 0,
"BlockWidth must be 32 or a multiple of 64.");
static_assert(BlockHeight % ChunkHeight == 0,
"BlockHeight must be a multiple of ChunkHeight.");
// ChunkHeight <= 64 keeps the int16_t accumulator in SubPixelVarianceWxHImp
// within bounds (at most 64 iterations * 2 diffs = 128 diffs per lane).
static_assert(ChunkHeight > 0 && ChunkHeight <= 64,
"ChunkHeight must be in (0, 64] to avoid int16_t overflow.");
static_assert((1 << (WLog2 + HLog2)) == BlockWidth * BlockHeight,
"(1 << (WLog2 + HLog2)) must equal BlockWidth * BlockHeight.");
HWY_IF_CONSTEXPR(BlockWidth == kChunkWidth && BlockHeight == ChunkHeight) {
const int sum = SubPixelVarianceWxHImp<kChunkWidth, HasSec>(
src, src_stride, x_offset, y_offset, dst, dst_stride,
HasSec ? sec : nullptr, HasSec ? BlockWidth : 0, BlockHeight, sse_ptr);
return *sse_ptr - static_cast<unsigned int>(
(static_cast<int64_t>(sum) * sum) >> (WLog2 + HLog2));
}
unsigned int sse = 0;
int se = 0;
for (int i = 0; i < BlockWidth / kChunkWidth; ++i) {
const uint8_t *src_ptr = src;
const uint8_t *dst_ptr = dst;
const uint8_t *sec_ptr = sec;
for (int j = 0; j < BlockHeight / ChunkHeight; ++j) {
unsigned int sse2;
const int se2 = SubPixelVarianceWxHImp<kChunkWidth, HasSec>(
src_ptr, src_stride, x_offset, y_offset, dst_ptr, dst_stride,
HasSec ? sec_ptr : nullptr, HasSec ? BlockWidth : 0, ChunkHeight,
&sse2);
dst_ptr += ChunkHeight * dst_stride;
src_ptr += ChunkHeight * src_stride;
if (HasSec) sec_ptr += ChunkHeight * BlockWidth;
se += se2;
sse += sse2;
}
src += kChunkWidth;
dst += kChunkWidth;
if (HasSec) sec += kChunkWidth;
}
*sse_ptr = sse;
return sse - static_cast<unsigned int>((static_cast<int64_t>(se) * se) >>
(WLog2 + HLog2));
}
// -----------------------------------------------------------------------------
// OBMC Variance & CompoundAvg / CompoundMask Pred
// -----------------------------------------------------------------------------
#if !CONFIG_REALTIME_ONLY
HWY_ATTR HWY_INLINE void ObmcVarianceW16N(
const uint8_t *pre, const int pre_stride, const int32_t *wsrc,
const int32_t *mask, unsigned int *const sse, int *const sum, const int w,
const int h) {
constexpr hn::ScalableTag<int32_t> di32;
constexpr hn::Repartition<int16_t, decltype(di32)> di16;
constexpr hn::Rebind<uint8_t, decltype(di32)> du8_quarter;
constexpr int kStep = hn::MaxLanes(di32);
assert(w >= kStep);
assert(IS_POWER_OF_TWO(w));
assert(IS_POWER_OF_TWO(h));
int n = 0;
const auto v_bias_d = hn::Set(di32, (1 << 12) >> 1);
auto v_sum_d = hn::Zero(di32);
auto v_sse_d = hn::Zero(di32);
for (int row = 0; row < h; ++row) {
for (int col = 0; col < w; col += kStep) {
const auto v_p_b = hn::LoadU(du8_quarter, pre + col);
const auto v_m_d = hn::LoadU(di32, mask + n);
const auto v_w_d = hn::LoadU(di32, wsrc + n);
const auto v_p_d = hn::PromoteTo(di32, v_p_b);
// Both pre ([0, 255]) and mask ([0, 64 * 64]) fit in signed 15-bit
// integers, so their upper int16_t halves in int32_t lanes are 0 and
// WidenMulPairwiseAdd computes exact 32-bit products pre * mask.
const auto v_pm_d = hn::WidenMulPairwiseAdd(
di32, hn::BitCast(di16, v_p_d), hn::BitCast(di16, v_m_d));
const auto v_diff_d = hn::Sub(v_w_d, v_pm_d);
const auto v_sign_d = hn::ShiftRight<31>(v_diff_d);
const auto v_tmp_d = hn::Add(hn::Add(v_diff_d, v_bias_d), v_sign_d);
const auto v_rdiff_d = hn::ShiftRight<12>(v_tmp_d);
v_sum_d = hn::Add(v_sum_d, v_rdiff_d);
v_sse_d = hn::Add(v_sse_d, hn::Mul(v_rdiff_d, v_rdiff_d));
n += kStep;
}
pre += pre_stride;
}
*sum = VarianceFinalFrom32BitSum(di32, v_sse_d, v_sum_d, sse);
}
#endif // !CONFIG_REALTIME_ONLY
HWY_ATTR HWY_INLINE void CompAvgPred(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride) {
constexpr hn::ScalableTag<uint8_t> du8;
constexpr int vw = hn::MaxLanes(du8);
int row = 0;
if (width == 8) {
assert(height % 4 == 0);
constexpr hn::FixedTag<uint8_t, 32> du8_32;
do {
const auto pred_0123 = hn::LoadU(du8_32, pred);
const auto ref_0123 = LoadRowsU<4>(du8_32, ref, ref_stride);
hn::StoreU(hn::AverageRound(pred_0123, ref_0123), du8_32, comp_pred);
row += 4;
pred += 32;
comp_pred += 32;
ref += 4 * ref_stride;
} while (row < height);
} else if (width == 16) {
constexpr int kRows = (16 < vw) ? (vw / 16) : 1;
do {
const auto pred_vec = hn::LoadU(du8, pred);
const auto ref_vec = LoadRowsU<kRows>(du8, ref, ref_stride);
hn::StoreU(hn::AverageRound(pred_vec, ref_vec), du8, comp_pred);
row += kRows;
pred += 16 * kRows;
comp_pred += 16 * kRows;
ref += kRows * ref_stride;
} while (row < height);
} else if (width == 32) {
constexpr int kRows = (32 < vw) ? (vw / 32) : 1;
do {
const auto pred_vec = hn::LoadU(du8, pred);
const auto ref_vec = LoadRowsU<kRows>(du8, ref, ref_stride);
hn::StoreU(hn::AverageRound(pred_vec, ref_vec), du8, comp_pred);
row += kRows;
pred += 32 * kRows;
comp_pred += 32 * kRows;
ref += kRows * ref_stride;
} while (row < height);
} else if (width % vw == 0) {
do {
for (int x = 0; x < width; x += vw) {
const auto p0 = hn::LoadU(du8, pred + x);
const auto r0 = hn::LoadU(du8, ref + x);
const auto p1 = hn::LoadU(du8, pred + width + x);
const auto r1 = hn::LoadU(du8, ref + ref_stride + x);
hn::StoreU(hn::AverageRound(p0, r0), du8, comp_pred + x);
hn::StoreU(hn::AverageRound(p1, r1), du8, comp_pred + width + x);
}
row += 2;
pred += 2 * width;
comp_pred += 2 * width;
ref += 2 * ref_stride;
} while (row < height);
} else {
aom_comp_avg_pred_c(comp_pred, pred, width, height, ref, ref_stride);
}
}
template <class DU8>
HWY_ATTR HWY_INLINE void CompMaskPredLine(DU8 du8, hn::VFromD<DU8> s0,
hn::VFromD<DU8> s1, hn::VFromD<DU8> a,
uint8_t *comp_pred) {
constexpr hn::Repartition<int8_t, DU8> di8;
constexpr hn::Repartition<int16_t, DU8> di16;
const auto alpha_max =
hn::Set(du8, static_cast<uint8_t>(AOM_BLEND_A64_MAX_ALPHA));
constexpr int16_t round_bits = 15 - AOM_BLEND_A64_ROUND_BITS;
const auto round_offset =
hn::Set(di16, static_cast<int16_t>(1 << round_bits));
const auto ma = hn::Sub(alpha_max, a);
const auto ssAL = hn::InterleaveLower(du8, s0, s1);
const auto aaAL = hn::BitCast(di8, hn::InterleaveLower(du8, a, ma));
const auto ssAH = hn::InterleaveUpper(du8, s0, s1);
const auto aaAH = hn::BitCast(di8, hn::InterleaveUpper(du8, a, ma));
const auto blendAL = hn::SatWidenMulPairwiseAdd(di16, ssAL, aaAL);
const auto blendAH = hn::SatWidenMulPairwiseAdd(di16, ssAH, aaAH);
const auto roundAL = hn::MulFixedPoint15(blendAL, round_offset);
const auto roundAH = hn::MulFixedPoint15(blendAH, round_offset);
const auto roundA = hn::ReorderDemote2To(du8, roundAL, roundAH);
hn::StoreU(roundA, du8, comp_pred);
}
HWY_ATTR HWY_INLINE void CompMaskPred(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride, const uint8_t *mask,
int mask_stride, int invert_mask) {
constexpr hn::ScalableTag<uint8_t> du8;
constexpr int vw = hn::MaxLanes(du8);
int i = 0;
const uint8_t *src0 = invert_mask ? pred : ref;
const uint8_t *src1 = invert_mask ? ref : pred;
const int stride0 = invert_mask ? width : ref_stride;
const int stride1 = invert_mask ? ref_stride : width;
if (width == 8) {
aom_comp_mask_pred_avx2(comp_pred, pred, width, height, ref, ref_stride,
mask, mask_stride, invert_mask);
} else if (width == 16) {
constexpr int kRows = (16 < vw) ? (vw / 16) : 1;
do {
const auto s0 = LoadRowsU<kRows>(du8, src0, stride0);
const auto s1 = LoadRowsU<kRows>(du8, src1, stride1);
const auto a = LoadRowsU<kRows>(du8, mask, mask_stride);
CompMaskPredLine(du8, s0, s1, a, comp_pred);
src0 += stride0 * kRows;
src1 += stride1 * kRows;
mask += mask_stride * kRows;
comp_pred += 16 * kRows;
i += kRows;
} while (i < height);
} else if (width == 32) {
constexpr int kRows = (32 < vw) ? (vw / 32) : 1;
do {
const auto s0 = LoadRowsU<kRows>(du8, src0, stride0);
const auto s1 = LoadRowsU<kRows>(du8, src1, stride1);
const auto a = LoadRowsU<kRows>(du8, mask, mask_stride);
CompMaskPredLine(du8, s0, s1, a, comp_pred);
src0 += stride0 * kRows;
src1 += stride1 * kRows;
mask += mask_stride * kRows;
comp_pred += 32 * kRows;
i += kRows;
} while (i < height);
} else {
do {
for (int x = 0; x < width; x += vw) {
const auto s0 = hn::LoadU(du8, src0 + x);
const auto s1 = hn::LoadU(du8, src1 + x);
const auto a = hn::LoadU(du8, mask + x);
CompMaskPredLine(du8, s0, s1, a, comp_pred);
comp_pred += vw;
}
src0 += stride0;
src1 += stride1;
mask += mask_stride;
i++;
} while (i < height);
}
}
#if CONFIG_AV1_HIGHBITDEPTH
template <class DI16>
HWY_ATTR HWY_INLINE hn::VFromD<DI16> HighbdCompMaskPredLine(
DI16 di16, hn::VFromD<DI16> s0, hn::VFromD<DI16> s1, hn::VFromD<DI16> a) {
constexpr hn::Repartition<int32_t, DI16> di32;
const auto alpha_max =
hn::Set(di16, static_cast<int16_t>(1 << AOM_BLEND_A64_ROUND_BITS));
const auto round_const = hn::Set(di32, (1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
const auto a_inv = hn::Sub(alpha_max, a);
const auto s_lo = hn::InterleaveLower(di16, s0, s1);
const auto a_lo = hn::InterleaveLower(di16, a, a_inv);
const auto pred_l = hn::ShiftRight<AOM_BLEND_A64_ROUND_BITS>(
hn::Add(hn::WidenMulPairwiseAdd(di32, s_lo, a_lo), round_const));
const auto s_hi = hn::InterleaveUpper(di16, s0, s1);
const auto a_hi = hn::InterleaveUpper(di16, a, a_inv);
const auto pred_h = hn::ShiftRight<AOM_BLEND_A64_ROUND_BITS>(
hn::Add(hn::WidenMulPairwiseAdd(di32, s_hi, a_hi), round_const));
return hn::ReorderDemote2To(di16, pred_l, pred_h);
}
HWY_ATTR HWY_INLINE void HighbdCompMaskPred(uint8_t *comp_pred8,
const uint8_t *pred8, int width,
int height, const uint8_t *ref8,
int ref_stride, const uint8_t *mask,
int mask_stride, int invert_mask) {
if (width < 16) {
aom_highbd_comp_mask_pred_avx2(comp_pred8, pred8, width, height, ref8,
ref_stride, mask, mask_stride, invert_mask);
return;
}
constexpr hn::ScalableTag<int16_t> di16;
constexpr hn::Rebind<uint8_t, decltype(di16)> du8_half;
constexpr int kLanes16 = hn::MaxLanes(di16);
const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
int16_t *comp_pred =
reinterpret_cast<int16_t *>(CONVERT_TO_SHORTPTR(comp_pred8));
const int16_t *src0 =
reinterpret_cast<const int16_t *>(invert_mask ? pred : ref);
const int16_t *src1 =
reinterpret_cast<const int16_t *>(invert_mask ? ref : pred);
const int stride0 = invert_mask ? width : ref_stride;
const int stride1 = invert_mask ? ref_stride : width;
if (width == 16) {
constexpr int kRows = (16 < kLanes16) ? (kLanes16 / 16) : 1;
for (int i = 0; i < height; i += kRows) {
const auto s0 = LoadRowsU<kRows>(di16, src0, stride0);
const auto s1 = LoadRowsU<kRows>(di16, src1, stride1);
const auto m8 = LoadRowsU<kRows>(du8_half, mask, mask_stride);
const auto m16 = hn::PromoteTo(di16, m8);
const auto comp = HighbdCompMaskPredLine(di16, s0, s1, m16);
hn::StoreU(comp, di16, comp_pred);
src0 += stride0 * kRows;
src1 += stride1 * kRows;
mask += mask_stride * kRows;
comp_pred += 16 * kRows;
}
} else {
for (int i = 0; i < height; ++i) {
for (int x = 0; x < width; x += kLanes16) {
const auto s0 = hn::LoadU(di16, src0 + x);
const auto s1 = hn::LoadU(di16, src1 + x);
const auto m8 = hn::LoadU(du8_half, mask + x);
const auto m16 = hn::PromoteTo(di16, m8);
const auto comp = HighbdCompMaskPredLine(di16, s0, s1, m16);
hn::StoreU(comp, di16, comp_pred);
comp_pred += kLanes16;
}
src0 += stride0;
src1 += stride1;
mask += mask_stride;
}
}
}
#endif // CONFIG_AV1_HIGHBITDEPTH
} // namespace HWY_NAMESPACE
} // namespace
// -----------------------------------------------------------------------------
// Export Macros
// -----------------------------------------------------------------------------
#define FVAR(w, h, bits, suffix) \
extern "C" unsigned int aom_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
unsigned int *sse); \
HWY_ATTR unsigned int aom_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
unsigned int *sse) { \
return HWY_NAMESPACE::VarianceWxH<w, h, bits>(src, src_stride, ref, \
ref_stride, sse); \
}
#define FMSE16X16(suffix) \
extern "C" unsigned int aom_mse16x16_##suffix( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
unsigned int *sse); \
HWY_ATTR unsigned int aom_mse16x16_##suffix( \
const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \
unsigned int *sse) { \
int sum = 0; \
HWY_NAMESPACE::GetBlockVar<16, 16>(src, src_stride, ref, ref_stride, sse, \
&sum); \
return *sse; \
}
#define FGET_BLK_SSE_SUM(suffix) \
extern "C" void aom_get_blk_sse_sum_##suffix(const int16_t *data, \
int stride, int bw, int bh, \
int *x_sum, int64_t *x2_sum); \
HWY_ATTR void aom_get_blk_sse_sum_##suffix(const int16_t *data, int stride, \
int bw, int bh, int *x_sum, \
int64_t *x2_sum) { \
HWY_NAMESPACE::GetBlkSseSum(data, stride, bw, bh, x_sum, x2_sum); \
}
#if CONFIG_AV1_HIGHBITDEPTH
#define FHIGHBD_CALC_VAR_STAT(suffix) \
extern "C" int64_t aom_highbd_calc_variance_stat_##suffix( \
const uint16_t *src, int stride, int bw, int bh); \
HWY_ATTR int64_t aom_highbd_calc_variance_stat_##suffix( \
const uint16_t *src, int stride, int bw, int bh) { \
return HWY_NAMESPACE::HighbdCalcVarianceStat(src, stride, bw, bh); \
}
#else
#define FHIGHBD_CALC_VAR_STAT(suffix)
#endif
#define FCALC_VAR_STAT(suffix) \
extern "C" int64_t aom_calc_variance_stat_##suffix( \
const uint8_t *src, int stride, int bw, int bh); \
HWY_ATTR int64_t aom_calc_variance_stat_##suffix( \
const uint8_t *src, int stride, int bw, int bh) { \
return HWY_NAMESPACE::CalcVarianceStat(src, stride, bw, bh); \
} \
FHIGHBD_CALC_VAR_STAT(suffix)
#if !CONFIG_REALTIME_ONLY
#define FOR_EACH_VAR_RECT_BLOCK_SIZE(X, suffix) \
X(64, 16, 10, suffix) \
X(32, 8, 8, suffix) \
X(16, 64, 10, suffix) \
X(16, 4, 6, suffix)
#else
#define FOR_EACH_VAR_RECT_BLOCK_SIZE(X, suffix)
#endif
#define FOR_EACH_VAR_BLOCK_SIZE(X, suffix) \
X(128, 128, 14, suffix) \
X(128, 64, 13, suffix) \
X(64, 128, 13, suffix) \
X(64, 64, 12, suffix) \
X(64, 32, 11, suffix) \
X(32, 64, 11, suffix) \
X(32, 32, 10, suffix) \
X(32, 16, 9, suffix) \
X(16, 32, 9, suffix) \
X(16, 16, 8, suffix) \
X(16, 8, 7, suffix) \
FOR_EACH_VAR_RECT_BLOCK_SIZE(X, suffix)
#define FSUBPIX_VAR_32XH(h, log2h, suffix) \
extern "C" unsigned int aom_sub_pixel_variance32x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse); \
HWY_ATTR unsigned int aom_sub_pixel_variance32x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse) { \
return HWY_NAMESPACE::SubPixelVarianceWxH<32, h, h, 5, log2h, false>( \
src, src_stride, x_offset, y_offset, dst, dst_stride, sse, nullptr); \
} \
extern "C" unsigned int aom_sub_pixel_avg_variance32x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse, \
const uint8_t *sec_ptr); \
HWY_ATTR unsigned int aom_sub_pixel_avg_variance32x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse, \
const uint8_t *sec_ptr) { \
return HWY_NAMESPACE::SubPixelVarianceWxH<32, h, h, 5, log2h, true>( \
src, src_stride, x_offset, y_offset, dst, dst_stride, sse, sec_ptr); \
}
#define FSUBPIX_VAR_64_128(w, h, hf, wlog2, hlog2, suffix) \
extern "C" unsigned int aom_sub_pixel_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse_ptr); \
HWY_ATTR unsigned int aom_sub_pixel_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse_ptr) { \
return HWY_NAMESPACE::SubPixelVarianceWxH<w, h, hf, wlog2, hlog2, false>( \
src, src_stride, x_offset, y_offset, dst, dst_stride, sse_ptr, \
nullptr); \
} \
extern "C" unsigned int aom_sub_pixel_avg_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse_ptr, \
const uint8_t *sec); \
HWY_ATTR unsigned int aom_sub_pixel_avg_variance##w##x##h##_##suffix( \
const uint8_t *src, int src_stride, int x_offset, int y_offset, \
const uint8_t *dst, int dst_stride, unsigned int *sse_ptr, \
const uint8_t *sec) { \
return HWY_NAMESPACE::SubPixelVarianceWxH<w, h, hf, wlog2, hlog2, true>( \
src, src_stride, x_offset, y_offset, dst, dst_stride, sse_ptr, sec); \
}
#if !CONFIG_REALTIME_ONLY
#define FOR_EACH_SUBPIX_VAR_32XH_RECT(X, suffix) X(8, 3, suffix)
#define FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix) X(64, 16, 16, 6, 4, suffix)
#else
#define FOR_EACH_SUBPIX_VAR_32XH_RECT(X, suffix)
#define FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix)
#endif
#define FOR_EACH_SUBPIX_VAR_32XH(X, suffix) \
X(64, 6, suffix) \
X(32, 5, suffix) \
X(16, 4, suffix) \
FOR_EACH_SUBPIX_VAR_32XH_RECT(X, suffix)
#define FOR_EACH_SUBPIX_VAR_64_128(X, suffix) \
X(128, 128, 64, 7, 7, suffix) \
X(128, 64, 64, 7, 6, suffix) \
X(64, 128, 64, 6, 7, suffix) \
X(64, 64, 64, 6, 6, suffix) \
X(64, 32, 32, 6, 5, suffix) \
FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix)
#if !CONFIG_REALTIME_ONLY
#define FOBMC_VAR(W, H, suffix) \
extern "C" unsigned int aom_obmc_variance##W##x##H##_##suffix( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse); \
HWY_ATTR unsigned int aom_obmc_variance##W##x##H##_##suffix( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
static_assert((W) >= 16, "FOBMC_VAR requires W >= 16."); \
int sum = 0; \
HWY_NAMESPACE::ObmcVarianceW16N(pre, pre_stride, wsrc, mask, sse, &sum, W, \
H); \
return *sse - static_cast<unsigned int>( \
(static_cast<int64_t>(sum) * sum) / ((W) * (H))); \
}
#define FOR_EACH_OBMC_VAR_BLOCK_SIZE(X, suffix) \
X(128, 128, suffix) \
X(128, 64, suffix) \
X(64, 128, suffix) \
X(64, 64, suffix) \
X(64, 32, suffix) \
X(64, 16, suffix) \
X(32, 64, suffix) \
X(32, 32, suffix) \
X(32, 16, suffix) \
X(32, 8, suffix) \
X(16, 64, suffix) \
X(16, 32, suffix) \
X(16, 16, suffix) \
X(16, 8, suffix) \
X(16, 4, suffix)
#else
#define FOBMC_VAR(W, H, suffix)
#define FOR_EACH_OBMC_VAR_BLOCK_SIZE(X, suffix)
#endif // !CONFIG_REALTIME_ONLY
#if CONFIG_AV1_HIGHBITDEPTH
#define FHIGHBD_COMP_MASK_PRED(suffix) \
extern "C" void aom_highbd_comp_mask_pred_##suffix( \
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, \
const uint8_t *ref8, int ref_stride, const uint8_t *mask, \
int mask_stride, int invert_mask); \
HWY_ATTR void aom_highbd_comp_mask_pred_##suffix( \
uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, \
const uint8_t *ref8, int ref_stride, const uint8_t *mask, \
int mask_stride, int invert_mask) { \
HWY_NAMESPACE::HighbdCompMaskPred(comp_pred8, pred8, width, height, ref8, \
ref_stride, mask, mask_stride, \
invert_mask); \
}
#else
#define FHIGHBD_COMP_MASK_PRED(suffix)
#endif
#define FCOMP_AVG_MASK_PRED(suffix) \
extern "C" void aom_comp_avg_pred_##suffix( \
uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
const uint8_t *ref, int ref_stride); \
HWY_ATTR void aom_comp_avg_pred_##suffix( \
uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
const uint8_t *ref, int ref_stride) { \
HWY_NAMESPACE::CompAvgPred(comp_pred, pred, width, height, ref, \
ref_stride); \
} \
extern "C" void aom_comp_mask_pred_##suffix( \
uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
const uint8_t *ref, int ref_stride, const uint8_t *mask, \
int mask_stride, int invert_mask); \
HWY_ATTR void aom_comp_mask_pred_##suffix( \
uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
const uint8_t *ref, int ref_stride, const uint8_t *mask, \
int mask_stride, int invert_mask) { \
HWY_NAMESPACE::CompMaskPred(comp_pred, pred, width, height, ref, \
ref_stride, mask, mask_stride, invert_mask); \
} \
FHIGHBD_COMP_MASK_PRED(suffix)
HWY_AFTER_NAMESPACE();
#endif // AOM_AOM_DSP_VARIANCE_HWY_H_