Highway: Add AVX512 obmc_var & comp_avg/mask_pred Function / Block Size │ AVX2 (ns) │ AVX512 (ns) │ Speedup (x) ────────────────────────┼───────────┼─────────────┼──────────── obmc_variance 16x8 │ 19.87 │ 11.37 │ 1.75x obmc_variance 16x16 │ 37.92 │ 19.57 │ 1.94x obmc_variance 16x32 │ 71.62 │ 45.00 │ 1.59x obmc_variance 32x16 │ 69.29 │ 41.55 │ 1.67x obmc_variance 32x32 │ 133.90 │ 76.72 │ 1.75x obmc_variance 32x64 │ 264.55 │ 145.39 │ 1.82x obmc_variance 64x32 │ 257.34 │ 140.64 │ 1.83x obmc_variance 64x64 │ 520.31 │ 283.27 │ 1.84x obmc_variance 64x128 │ 1028.12 │ 622.77 │ 1.65x obmc_variance 128x64 │ 1024.66 │ 565.44 │ 1.81x obmc_variance 128x128 │ 2015.57 │ 1072.11 │ 1.88x comp_avg_pred 32x32 │ 12.57 │ 8.16 │ 1.54x comp_avg_pred 64x64 │ 58.28 │ 31.46 │ 1.85x comp_avg_pred 128x128 │ 204.89 │ 96.21 │ 2.13x comp_mask_pred 32x32 │ 40.82 │ 23.45 │ 1.74x comp_mask_pred 64x64 │ 111.31 │ 67.04 │ 1.66x comp_mask_pred 128x128 │ 457.80 │ 379.88 │ 1.21x Change-Id: Ib00bfd0c986bf1cc92fa90a0021b55c5ec775873
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 7358e4d..3c38010 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1837,6 +1837,9 @@ add_proto qw/unsigned int/, "aom_obmc_variance${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, unsigned int *sse"; add_proto qw/unsigned int/, "aom_obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, unsigned int *sse"; specialize "aom_obmc_variance${w}x${h}", qw/sse4_1 avx2 neon/; + if (aom_config("CONFIG_HIGHWAY") eq "yes" && $w >= 16) { + specialize "aom_obmc_variance${w}x${h}", qw/avx512/; + } specialize "aom_obmc_sub_pixel_variance${w}x${h}", qw/sse4_1 neon/; } @@ -1858,6 +1861,9 @@ # add_proto qw/void aom_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride"; specialize qw/aom_comp_avg_pred avx2 neon/; + if (aom_config("CONFIG_HIGHWAY") eq "yes") { + specialize qw/aom_comp_avg_pred avx512/; + } if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { add_proto qw/void aom_highbd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride"; @@ -1869,10 +1875,16 @@ add_proto qw/void aom_comp_mask_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask"; specialize qw/aom_comp_mask_pred ssse3 avx2 neon/; + if (aom_config("CONFIG_HIGHWAY") eq "yes") { + specialize qw/aom_comp_mask_pred avx512/; + } if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { add_proto qw/void aom_highbd_comp_mask_pred/, "uint8_t *comp_pred, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask"; specialize qw/aom_highbd_comp_mask_pred sse2 avx2 neon/; + if (aom_config("CONFIG_HIGHWAY") eq "yes") { + specialize qw/aom_highbd_comp_mask_pred avx512/; + } } # Flow estimation library
diff --git a/aom_dsp/variance_hwy.h b/aom_dsp/variance_hwy.h index 4d53315..6ce320e 100644 --- a/aom_dsp/variance_hwy.h +++ b/aom_dsp/variance_hwy.h
@@ -19,6 +19,8 @@ #include "config/aom_config.h" #include "config/aom_dsp_rtcd.h" +#include "aom_dsp/aom_dsp_common.h" +#include "aom_dsp/blend.h" #include "aom_dsp/reduce_sum_hwy.h" #include "third_party/highway/hwy/highway.h" @@ -704,6 +706,281 @@ (WLog2 + HLog2)); } +// ----------------------------------------------------------------------------- +// OBMC Variance & CompoundAvg / CompoundMask Pred +// ----------------------------------------------------------------------------- + +#if !CONFIG_REALTIME_ONLY +HWY_ATTR HWY_INLINE void ObmcVarianceW16N( + const uint8_t *pre, const int pre_stride, const int32_t *wsrc, + const int32_t *mask, unsigned int *const sse, int *const sum, const int w, + const int h) { + constexpr hn::ScalableTag<int32_t> di32; + constexpr hn::Repartition<int16_t, decltype(di32)> di16; + constexpr hn::Rebind<uint8_t, decltype(di32)> du8_quarter; + constexpr int kStep = hn::MaxLanes(di32); + + assert(w >= kStep); + assert(IS_POWER_OF_TWO(w)); + assert(IS_POWER_OF_TWO(h)); + + int n = 0; + const auto v_bias_d = hn::Set(di32, (1 << 12) >> 1); + auto v_sum_d = hn::Zero(di32); + auto v_sse_d = hn::Zero(di32); + + for (int row = 0; row < h; ++row) { + for (int col = 0; col < w; col += kStep) { + const auto v_p_b = hn::LoadU(du8_quarter, pre + col); + const auto v_m_d = hn::LoadU(di32, mask + n); + const auto v_w_d = hn::LoadU(di32, wsrc + n); + const auto v_p_d = hn::PromoteTo(di32, v_p_b); + // Both pre ([0, 255]) and mask ([0, 64 * 64]) fit in signed 15-bit + // integers, so their upper int16_t halves in int32_t lanes are 0 and + // WidenMulPairwiseAdd computes exact 32-bit products pre * mask. + const auto v_pm_d = hn::WidenMulPairwiseAdd( + di32, hn::BitCast(di16, v_p_d), hn::BitCast(di16, v_m_d)); + const auto v_diff_d = hn::Sub(v_w_d, v_pm_d); + const auto v_sign_d = hn::ShiftRight<31>(v_diff_d); + const auto v_tmp_d = hn::Add(hn::Add(v_diff_d, v_bias_d), v_sign_d); + const auto v_rdiff_d = hn::ShiftRight<12>(v_tmp_d); + + v_sum_d = hn::Add(v_sum_d, v_rdiff_d); + v_sse_d = hn::Add(v_sse_d, hn::Mul(v_rdiff_d, v_rdiff_d)); + n += kStep; + } + pre += pre_stride; + } + *sum = VarianceFinalFrom32BitSum(di32, v_sse_d, v_sum_d, sse); +} +#endif // !CONFIG_REALTIME_ONLY + +HWY_ATTR HWY_INLINE void CompAvgPred(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride) { + constexpr hn::ScalableTag<uint8_t> du8; + constexpr int vw = hn::MaxLanes(du8); + int row = 0; + if (width == 8) { + assert(height % 4 == 0); + constexpr hn::FixedTag<uint8_t, 32> du8_32; + do { + const auto pred_0123 = hn::LoadU(du8_32, pred); + const auto ref_0123 = LoadRowsU<4>(du8_32, ref, ref_stride); + hn::StoreU(hn::AverageRound(pred_0123, ref_0123), du8_32, comp_pred); + row += 4; + pred += 32; + comp_pred += 32; + ref += 4 * ref_stride; + } while (row < height); + } else if (width == 16) { + constexpr int kRows = (16 < vw) ? (vw / 16) : 1; + do { + const auto pred_vec = hn::LoadU(du8, pred); + const auto ref_vec = LoadRowsU<kRows>(du8, ref, ref_stride); + hn::StoreU(hn::AverageRound(pred_vec, ref_vec), du8, comp_pred); + row += kRows; + pred += 16 * kRows; + comp_pred += 16 * kRows; + ref += kRows * ref_stride; + } while (row < height); + } else if (width == 32) { + constexpr int kRows = (32 < vw) ? (vw / 32) : 1; + do { + const auto pred_vec = hn::LoadU(du8, pred); + const auto ref_vec = LoadRowsU<kRows>(du8, ref, ref_stride); + hn::StoreU(hn::AverageRound(pred_vec, ref_vec), du8, comp_pred); + row += kRows; + pred += 32 * kRows; + comp_pred += 32 * kRows; + ref += kRows * ref_stride; + } while (row < height); + } else if (width % vw == 0) { + do { + for (int x = 0; x < width; x += vw) { + const auto p0 = hn::LoadU(du8, pred + x); + const auto r0 = hn::LoadU(du8, ref + x); + const auto p1 = hn::LoadU(du8, pred + width + x); + const auto r1 = hn::LoadU(du8, ref + ref_stride + x); + hn::StoreU(hn::AverageRound(p0, r0), du8, comp_pred + x); + hn::StoreU(hn::AverageRound(p1, r1), du8, comp_pred + width + x); + } + row += 2; + pred += 2 * width; + comp_pred += 2 * width; + ref += 2 * ref_stride; + } while (row < height); + } else { + aom_comp_avg_pred_c(comp_pred, pred, width, height, ref, ref_stride); + } +} + +template <class DU8> +HWY_ATTR HWY_INLINE void CompMaskPredLine(DU8 du8, hn::VFromD<DU8> s0, + hn::VFromD<DU8> s1, hn::VFromD<DU8> a, + uint8_t *comp_pred) { + constexpr hn::Repartition<int8_t, DU8> di8; + constexpr hn::Repartition<int16_t, DU8> di16; + const auto alpha_max = + hn::Set(du8, static_cast<uint8_t>(AOM_BLEND_A64_MAX_ALPHA)); + constexpr int16_t round_bits = 15 - AOM_BLEND_A64_ROUND_BITS; + const auto round_offset = + hn::Set(di16, static_cast<int16_t>(1 << round_bits)); + const auto ma = hn::Sub(alpha_max, a); + + const auto ssAL = hn::InterleaveLower(du8, s0, s1); + const auto aaAL = hn::BitCast(di8, hn::InterleaveLower(du8, a, ma)); + const auto ssAH = hn::InterleaveUpper(du8, s0, s1); + const auto aaAH = hn::BitCast(di8, hn::InterleaveUpper(du8, a, ma)); + + const auto blendAL = hn::SatWidenMulPairwiseAdd(di16, ssAL, aaAL); + const auto blendAH = hn::SatWidenMulPairwiseAdd(di16, ssAH, aaAH); + const auto roundAL = hn::MulFixedPoint15(blendAL, round_offset); + const auto roundAH = hn::MulFixedPoint15(blendAH, round_offset); + + const auto roundA = hn::ReorderDemote2To(du8, roundAL, roundAH); + hn::StoreU(roundA, du8, comp_pred); +} + +HWY_ATTR HWY_INLINE void CompMaskPred(uint8_t *comp_pred, const uint8_t *pred, + int width, int height, const uint8_t *ref, + int ref_stride, const uint8_t *mask, + int mask_stride, int invert_mask) { + constexpr hn::ScalableTag<uint8_t> du8; + constexpr int vw = hn::MaxLanes(du8); + int i = 0; + const uint8_t *src0 = invert_mask ? pred : ref; + const uint8_t *src1 = invert_mask ? ref : pred; + const int stride0 = invert_mask ? width : ref_stride; + const int stride1 = invert_mask ? ref_stride : width; + + if (width == 8) { + aom_comp_mask_pred_avx2(comp_pred, pred, width, height, ref, ref_stride, + mask, mask_stride, invert_mask); + } else if (width == 16) { + constexpr int kRows = (16 < vw) ? (vw / 16) : 1; + do { + const auto s0 = LoadRowsU<kRows>(du8, src0, stride0); + const auto s1 = LoadRowsU<kRows>(du8, src1, stride1); + const auto a = LoadRowsU<kRows>(du8, mask, mask_stride); + CompMaskPredLine(du8, s0, s1, a, comp_pred); + src0 += stride0 * kRows; + src1 += stride1 * kRows; + mask += mask_stride * kRows; + comp_pred += 16 * kRows; + i += kRows; + } while (i < height); + } else if (width == 32) { + constexpr int kRows = (32 < vw) ? (vw / 32) : 1; + do { + const auto s0 = LoadRowsU<kRows>(du8, src0, stride0); + const auto s1 = LoadRowsU<kRows>(du8, src1, stride1); + const auto a = LoadRowsU<kRows>(du8, mask, mask_stride); + CompMaskPredLine(du8, s0, s1, a, comp_pred); + src0 += stride0 * kRows; + src1 += stride1 * kRows; + mask += mask_stride * kRows; + comp_pred += 32 * kRows; + i += kRows; + } while (i < height); + } else { + do { + for (int x = 0; x < width; x += vw) { + const auto s0 = hn::LoadU(du8, src0 + x); + const auto s1 = hn::LoadU(du8, src1 + x); + const auto a = hn::LoadU(du8, mask + x); + CompMaskPredLine(du8, s0, s1, a, comp_pred); + comp_pred += vw; + } + src0 += stride0; + src1 += stride1; + mask += mask_stride; + i++; + } while (i < height); + } +} + +#if CONFIG_AV1_HIGHBITDEPTH +template <class DI16> +HWY_ATTR HWY_INLINE hn::VFromD<DI16> HighbdCompMaskPredLine( + DI16 di16, hn::VFromD<DI16> s0, hn::VFromD<DI16> s1, hn::VFromD<DI16> a) { + constexpr hn::Repartition<int32_t, DI16> di32; + const auto alpha_max = + hn::Set(di16, static_cast<int16_t>(1 << AOM_BLEND_A64_ROUND_BITS)); + const auto round_const = hn::Set(di32, (1 << AOM_BLEND_A64_ROUND_BITS) >> 1); + const auto a_inv = hn::Sub(alpha_max, a); + + const auto s_lo = hn::InterleaveLower(di16, s0, s1); + const auto a_lo = hn::InterleaveLower(di16, a, a_inv); + const auto pred_l = hn::ShiftRight<AOM_BLEND_A64_ROUND_BITS>( + hn::Add(hn::WidenMulPairwiseAdd(di32, s_lo, a_lo), round_const)); + + const auto s_hi = hn::InterleaveUpper(di16, s0, s1); + const auto a_hi = hn::InterleaveUpper(di16, a, a_inv); + const auto pred_h = hn::ShiftRight<AOM_BLEND_A64_ROUND_BITS>( + hn::Add(hn::WidenMulPairwiseAdd(di32, s_hi, a_hi), round_const)); + + return hn::ReorderDemote2To(di16, pred_l, pred_h); +} + +HWY_ATTR HWY_INLINE void HighbdCompMaskPred(uint8_t *comp_pred8, + const uint8_t *pred8, int width, + int height, const uint8_t *ref8, + int ref_stride, const uint8_t *mask, + int mask_stride, int invert_mask) { + if (width < 16) { + aom_highbd_comp_mask_pred_avx2(comp_pred8, pred8, width, height, ref8, + ref_stride, mask, mask_stride, invert_mask); + return; + } + constexpr hn::ScalableTag<int16_t> di16; + constexpr hn::Rebind<uint8_t, decltype(di16)> du8_half; + constexpr int kLanes16 = hn::MaxLanes(di16); + + const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8); + const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); + int16_t *comp_pred = + reinterpret_cast<int16_t *>(CONVERT_TO_SHORTPTR(comp_pred8)); + const int16_t *src0 = + reinterpret_cast<const int16_t *>(invert_mask ? pred : ref); + const int16_t *src1 = + reinterpret_cast<const int16_t *>(invert_mask ? ref : pred); + const int stride0 = invert_mask ? width : ref_stride; + const int stride1 = invert_mask ? ref_stride : width; + + if (width == 16) { + constexpr int kRows = (16 < kLanes16) ? (kLanes16 / 16) : 1; + for (int i = 0; i < height; i += kRows) { + const auto s0 = LoadRowsU<kRows>(di16, src0, stride0); + const auto s1 = LoadRowsU<kRows>(di16, src1, stride1); + const auto m8 = LoadRowsU<kRows>(du8_half, mask, mask_stride); + const auto m16 = hn::PromoteTo(di16, m8); + const auto comp = HighbdCompMaskPredLine(di16, s0, s1, m16); + hn::StoreU(comp, di16, comp_pred); + src0 += stride0 * kRows; + src1 += stride1 * kRows; + mask += mask_stride * kRows; + comp_pred += 16 * kRows; + } + } else { + for (int i = 0; i < height; ++i) { + for (int x = 0; x < width; x += kLanes16) { + const auto s0 = hn::LoadU(di16, src0 + x); + const auto s1 = hn::LoadU(di16, src1 + x); + const auto m8 = hn::LoadU(du8_half, mask + x); + const auto m16 = hn::PromoteTo(di16, m8); + const auto comp = HighbdCompMaskPredLine(di16, s0, s1, m16); + hn::StoreU(comp, di16, comp_pred); + comp_pred += kLanes16; + } + src0 += stride0; + src1 += stride1; + mask += mask_stride; + } + } +} +#endif // CONFIG_AV1_HIGHBITDEPTH + } // namespace HWY_NAMESPACE } // namespace @@ -857,6 +1134,84 @@ X(64, 32, 32, 6, 5, suffix) \ FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix) +#if !CONFIG_REALTIME_ONLY +#define FOBMC_VAR(W, H, suffix) \ + extern "C" unsigned int aom_obmc_variance##W##x##H##_##suffix( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse); \ + HWY_ATTR unsigned int aom_obmc_variance##W##x##H##_##suffix( \ + const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ + const int32_t *mask, unsigned int *sse) { \ + static_assert((W) >= 16, "FOBMC_VAR requires W >= 16."); \ + int sum = 0; \ + HWY_NAMESPACE::ObmcVarianceW16N(pre, pre_stride, wsrc, mask, sse, &sum, W, \ + H); \ + return *sse - static_cast<unsigned int>( \ + (static_cast<int64_t>(sum) * sum) / ((W) * (H))); \ + } + +#define FOR_EACH_OBMC_VAR_BLOCK_SIZE(X, suffix) \ + X(128, 128, suffix) \ + X(128, 64, suffix) \ + X(64, 128, suffix) \ + X(64, 64, suffix) \ + X(64, 32, suffix) \ + X(64, 16, suffix) \ + X(32, 64, suffix) \ + X(32, 32, suffix) \ + X(32, 16, suffix) \ + X(32, 8, suffix) \ + X(16, 64, suffix) \ + X(16, 32, suffix) \ + X(16, 16, suffix) \ + X(16, 8, suffix) \ + X(16, 4, suffix) +#else +#define FOBMC_VAR(W, H, suffix) +#define FOR_EACH_OBMC_VAR_BLOCK_SIZE(X, suffix) +#endif // !CONFIG_REALTIME_ONLY + +#if CONFIG_AV1_HIGHBITDEPTH +#define FHIGHBD_COMP_MASK_PRED(suffix) \ + extern "C" void aom_highbd_comp_mask_pred_##suffix( \ + uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, \ + const uint8_t *ref8, int ref_stride, const uint8_t *mask, \ + int mask_stride, int invert_mask); \ + HWY_ATTR void aom_highbd_comp_mask_pred_##suffix( \ + uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, \ + const uint8_t *ref8, int ref_stride, const uint8_t *mask, \ + int mask_stride, int invert_mask) { \ + HWY_NAMESPACE::HighbdCompMaskPred(comp_pred8, pred8, width, height, ref8, \ + ref_stride, mask, mask_stride, \ + invert_mask); \ + } +#else +#define FHIGHBD_COMP_MASK_PRED(suffix) +#endif + +#define FCOMP_AVG_MASK_PRED(suffix) \ + extern "C" void aom_comp_avg_pred_##suffix( \ + uint8_t *comp_pred, const uint8_t *pred, int width, int height, \ + const uint8_t *ref, int ref_stride); \ + HWY_ATTR void aom_comp_avg_pred_##suffix( \ + uint8_t *comp_pred, const uint8_t *pred, int width, int height, \ + const uint8_t *ref, int ref_stride) { \ + HWY_NAMESPACE::CompAvgPred(comp_pred, pred, width, height, ref, \ + ref_stride); \ + } \ + extern "C" void aom_comp_mask_pred_##suffix( \ + uint8_t *comp_pred, const uint8_t *pred, int width, int height, \ + const uint8_t *ref, int ref_stride, const uint8_t *mask, \ + int mask_stride, int invert_mask); \ + HWY_ATTR void aom_comp_mask_pred_##suffix( \ + uint8_t *comp_pred, const uint8_t *pred, int width, int height, \ + const uint8_t *ref, int ref_stride, const uint8_t *mask, \ + int mask_stride, int invert_mask) { \ + HWY_NAMESPACE::CompMaskPred(comp_pred, pred, width, height, ref, \ + ref_stride, mask, mask_stride, invert_mask); \ + } \ + FHIGHBD_COMP_MASK_PRED(suffix) + HWY_AFTER_NAMESPACE(); #endif // AOM_AOM_DSP_VARIANCE_HWY_H_
diff --git a/aom_dsp/x86/variance_hwy_avx512.cc b/aom_dsp/x86/variance_hwy_avx512.cc index d9ab225..fd88856 100644 --- a/aom_dsp/x86/variance_hwy_avx512.cc +++ b/aom_dsp/x86/variance_hwy_avx512.cc
@@ -21,3 +21,6 @@ FOR_EACH_SUBPIX_VAR_32XH(FSUBPIX_VAR_32XH, avx512) FOR_EACH_SUBPIX_VAR_64_128(FSUBPIX_VAR_64_128, avx512) + +FOR_EACH_OBMC_VAR_BLOCK_SIZE(FOBMC_VAR, avx512) +FCOMP_AVG_MASK_PRED(avx512)
diff --git a/test/comp_mask_pred_test.cc b/test/comp_mask_pred_test.cc index be6a1fb..0f1b529 100644 --- a/test/comp_mask_pred_test.cc +++ b/test/comp_mask_pred_test.cc
@@ -18,6 +18,7 @@ #include "aom/aom_codec.h" #include "aom/aom_integer.h" +#include "aom_dsp/blend.h" #include "aom_dsp/variance.h" #include "aom_mem/aom_mem.h" #include "aom_ports/aom_timer.h" @@ -39,10 +40,12 @@ int width, int height, const uint8_t *ref, int ref_stride); -#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_NEON +#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_AVX512 || HAVE_NEON const BLOCK_SIZE kCompMaskPredParams[] = { - BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8, BLOCK_16X16, - BLOCK_16X32, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32 + BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8, BLOCK_16X16, + BLOCK_16X32, BLOCK_16X64, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32, + BLOCK_32X64, BLOCK_64X16, BLOCK_64X32, BLOCK_64X64, BLOCK_64X128, + BLOCK_128X64, BLOCK_128X128 }; #endif @@ -128,15 +131,27 @@ const int w = block_size_wide[bsize]; const int h = block_size_high[bsize]; const int wedge_types = get_wedge_types_lookup(bsize); - for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { - const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); + if (wedge_types > 0) { + for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { + const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); + aom_comp_mask_pred_c(comp_pred1_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w, + inv); + test_impl(comp_pred2_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w, inv); + + ASSERT_EQ(CheckResult(w, h), true) + << " wedge " << wedge_index << " inv " << inv; + } + } else { + DECLARE_ALIGNED(16, uint8_t, mask[MAX_SB_SQUARE]); + for (int i = 0; i < w * h; ++i) { + mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1); + } aom_comp_mask_pred_c(comp_pred1_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w, inv); test_impl(comp_pred2_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w, inv); - ASSERT_EQ(CheckResult(w, h), true) - << " wedge " << wedge_index << " inv " << inv; + ASSERT_EQ(CheckResult(w, h), true) << " random mask inv " << inv; } } @@ -145,8 +160,16 @@ const int w = block_size_wide[bsize]; const int h = block_size_high[bsize]; const int wedge_types = get_wedge_types_lookup(bsize); - int wedge_index = wedge_types / 2; - const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); + DECLARE_ALIGNED(16, uint8_t, rand_mask[MAX_SB_SQUARE]); + const uint8_t *mask = nullptr; + if (wedge_types > 0) { + mask = av1_get_contiguous_soft_mask(wedge_types / 2, 1, bsize); + } else { + for (int i = 0; i < w * h; ++i) { + rand_mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1); + } + mask = rand_mask; + } const int num_loops = 1000000000 / (w + h); comp_mask_pred_func funcs[2] = { aom_comp_mask_pred_c, test_impl }; @@ -193,6 +216,13 @@ ::testing::ValuesIn(kCompMaskPredParams))); #endif +#if CONFIG_HIGHWAY && HAVE_AVX512 +INSTANTIATE_TEST_SUITE_P( + AVX512, AV1CompMaskPredTest, + ::testing::Combine(::testing::Values(&aom_comp_mask_pred_avx512), + ::testing::ValuesIn(kCompMaskPredParams))); +#endif + #if HAVE_NEON INSTANTIATE_TEST_SUITE_P( NEON, AV1CompMaskPredTest, @@ -200,7 +230,7 @@ ::testing::ValuesIn(kCompMaskPredParams))); #endif -#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_NEON +#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_AVX512 || HAVE_NEON const BLOCK_SIZE kValidBlockSize[] = { BLOCK_4X4, BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8, BLOCK_16X16, BLOCK_16X32, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32, @@ -413,6 +443,13 @@ ::testing::ValuesIn(kValidBlockSize))); #endif +#if CONFIG_HIGHWAY && HAVE_AVX512 +INSTANTIATE_TEST_SUITE_P( + AVX512, AV1CompAvgPredTest, + ::testing::Combine(::testing::Values(&aom_comp_avg_pred_avx512), + ::testing::ValuesIn(kValidBlockSize))); +#endif + #if HAVE_NEON INSTANTIATE_TEST_SUITE_P( NEON, AV1CompAvgPredTest, @@ -520,9 +557,25 @@ ref_buffer_[i] = rnd_.Rand16() & ((1 << bd_) - 1); } - for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { - const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); + if (wedge_types > 0) { + for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) { + const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); + aom_highbd_comp_mask_pred_c( + CONVERT_TO_BYTEPTR(comp_pred1_), CONVERT_TO_BYTEPTR(pred_), w, h, + CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv); + + test_impl(CONVERT_TO_BYTEPTR(comp_pred2_), CONVERT_TO_BYTEPTR(pred_), w, + h, CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv); + + ASSERT_EQ(CheckResult(w, h), true) + << " wedge " << wedge_index << " inv " << inv; + } + } else { + DECLARE_ALIGNED(16, uint8_t, mask[MAX_SB_SQUARE]); + for (int i = 0; i < w * h; ++i) { + mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1); + } aom_highbd_comp_mask_pred_c( CONVERT_TO_BYTEPTR(comp_pred1_), CONVERT_TO_BYTEPTR(pred_), w, h, CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv); @@ -530,8 +583,7 @@ test_impl(CONVERT_TO_BYTEPTR(comp_pred2_), CONVERT_TO_BYTEPTR(pred_), w, h, CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv); - ASSERT_EQ(CheckResult(w, h), true) - << " wedge " << wedge_index << " inv " << inv; + ASSERT_EQ(CheckResult(w, h), true) << " random mask inv " << inv; } } @@ -542,7 +594,6 @@ const int w = block_size_wide[bsize]; const int h = block_size_high[bsize]; const int wedge_types = get_wedge_types_lookup(bsize); - int wedge_index = wedge_types / 2; for (int i = 0; i < MAX_SB_SQUARE; ++i) { pred_[i] = rnd_.Rand16() & ((1 << bd_) - 1); @@ -551,7 +602,16 @@ ref_buffer_[i] = rnd_.Rand16() & ((1 << bd_) - 1); } - const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize); + DECLARE_ALIGNED(16, uint8_t, rand_mask[MAX_SB_SQUARE]); + const uint8_t *mask = nullptr; + if (wedge_types > 0) { + mask = av1_get_contiguous_soft_mask(wedge_types / 2, 1, bsize); + } else { + for (int i = 0; i < w * h; ++i) { + rand_mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1); + } + mask = rand_mask; + } const int num_loops = 1000000000 / (w + h); highbd_comp_mask_pred_func funcs[2] = { aom_highbd_comp_mask_pred_c, @@ -602,6 +662,14 @@ ::testing::Range(8, 13, 2))); #endif +#if CONFIG_HIGHWAY && HAVE_AVX512 +INSTANTIATE_TEST_SUITE_P( + AVX512, AV1HighbdCompMaskPredTest, + ::testing::Combine(::testing::Values(&aom_highbd_comp_mask_pred_avx512), + ::testing::ValuesIn(kCompMaskPredParams), + ::testing::Range(8, 13, 2))); +#endif + #if HAVE_SSE2 INSTANTIATE_TEST_SUITE_P( SSE2, AV1HighbdCompMaskPredTest,
diff --git a/test/obmc_variance_test.cc b/test/obmc_variance_test.cc index 16482f7..23d7fa5 100644 --- a/test/obmc_variance_test.cc +++ b/test/obmc_variance_test.cc
@@ -194,6 +194,30 @@ ::testing::ValuesIn(avx2_functions)); #endif // HAVE_AVX2 +#if CONFIG_HIGHWAY && HAVE_AVX512 +const ObmcVarianceTest::ParamType avx512_functions[] = { + TestFuncs(aom_obmc_variance128x128_c, aom_obmc_variance128x128_avx512), + TestFuncs(aom_obmc_variance128x64_c, aom_obmc_variance128x64_avx512), + TestFuncs(aom_obmc_variance64x128_c, aom_obmc_variance64x128_avx512), + TestFuncs(aom_obmc_variance64x64_c, aom_obmc_variance64x64_avx512), + TestFuncs(aom_obmc_variance64x32_c, aom_obmc_variance64x32_avx512), + TestFuncs(aom_obmc_variance32x64_c, aom_obmc_variance32x64_avx512), + TestFuncs(aom_obmc_variance32x32_c, aom_obmc_variance32x32_avx512), + TestFuncs(aom_obmc_variance32x16_c, aom_obmc_variance32x16_avx512), + TestFuncs(aom_obmc_variance16x32_c, aom_obmc_variance16x32_avx512), + TestFuncs(aom_obmc_variance16x16_c, aom_obmc_variance16x16_avx512), + TestFuncs(aom_obmc_variance16x8_c, aom_obmc_variance16x8_avx512), + + TestFuncs(aom_obmc_variance64x16_c, aom_obmc_variance64x16_avx512), + TestFuncs(aom_obmc_variance16x64_c, aom_obmc_variance16x64_avx512), + TestFuncs(aom_obmc_variance32x8_c, aom_obmc_variance32x8_avx512), + TestFuncs(aom_obmc_variance16x4_c, aom_obmc_variance16x4_avx512), +}; + +INSTANTIATE_TEST_SUITE_P(AVX512, ObmcVarianceTest, + ::testing::ValuesIn(avx512_functions)); +#endif // CONFIG_HIGHWAY && HAVE_AVX512 + #if HAVE_NEON const ObmcVarianceTest::ParamType neon_functions[] = { TestFuncs(aom_obmc_variance128x128_c, aom_obmc_variance128x128_neon),