Add SSE4.1 vpx_obmc_variance* implementations and cosmetics Speedup for these functions: 4x Also include some cosmetic changes to SAD functions Change-Id: I344c32c795492507ae08742f52d035a13f583799
diff --git a/test/obmc_variance_test.cc b/test/obmc_variance_test.cc new file mode 100644 index 0000000..40295f2 --- /dev/null +++ b/test/obmc_variance_test.cc
@@ -0,0 +1,292 @@ +/* + * Copyright (c) 2016 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/acm_random.h" + +#include "test/function_equivalence_test.h" +#include "test/register_state_check.h" + +#include "./vpx_config.h" +#include "./vpx_dsp_rtcd.h" +#include "vpx/vpx_integer.h" + +#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE) + +using libvpx_test::ACMRandom; +using libvpx_test::FunctionEquivalenceTest; + +namespace { + +static const int kIterations = 1000; +static const int kMaskMax = 64; + +typedef unsigned int (*ObmcVarF)(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + unsigned int *sse); +typedef libvpx_test::FuncParam<ObmcVarF> TestFuncs; + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +class ObmcVarianceTest : public FunctionEquivalenceTest<ObmcVarF> {}; + +TEST_P(ObmcVarianceTest, RandomValues) { + DECLARE_ALIGNED(32, uint8_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) { + const int pre_stride = this->rng_(MAX_SB_SIZE + 1); + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + pre[i] = this->rng_.Rand8(); + wsrc[i] = this->rng_.Rand8() * this->rng_(kMaskMax * kMaskMax + 1); + mask[i] = this->rng_(kMaskMax * kMaskMax + 1); + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = params_.ref_func(pre, pre_stride, wsrc, mask, + &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = params_.tst_func(pre, pre_stride, wsrc, mask, &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +TEST_P(ObmcVarianceTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint8_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) { + const int pre_stride = iter; + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + pre[i] = UINT8_MAX; + wsrc[i] = UINT8_MAX * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = params_.ref_func(pre, pre_stride, wsrc, mask, + &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = params_.tst_func(pre, pre_stride, wsrc, mask, &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +#if HAVE_SSE4_1 +const ObmcVarianceTest::ParamType sse4_functions[] = { +#if CONFIG_EXT_PARTITION + TestFuncs(vpx_obmc_variance128x128_c, vpx_obmc_variance128x128_sse4_1), + TestFuncs(vpx_obmc_variance128x64_c, vpx_obmc_variance128x64_sse4_1), + TestFuncs(vpx_obmc_variance64x128_c, vpx_obmc_variance64x128_sse4_1), +#endif // CONFIG_EXT_PARTITION + TestFuncs(vpx_obmc_variance64x64_c, vpx_obmc_variance64x64_sse4_1), + TestFuncs(vpx_obmc_variance64x32_c, vpx_obmc_variance64x32_sse4_1), + TestFuncs(vpx_obmc_variance32x64_c, vpx_obmc_variance32x64_sse4_1), + TestFuncs(vpx_obmc_variance32x32_c, vpx_obmc_variance32x32_sse4_1), + TestFuncs(vpx_obmc_variance32x16_c, vpx_obmc_variance32x16_sse4_1), + TestFuncs(vpx_obmc_variance16x32_c, vpx_obmc_variance16x32_sse4_1), + TestFuncs(vpx_obmc_variance16x16_c, vpx_obmc_variance16x16_sse4_1), + TestFuncs(vpx_obmc_variance16x8_c, vpx_obmc_variance16x8_sse4_1), + TestFuncs(vpx_obmc_variance8x16_c, vpx_obmc_variance8x16_sse4_1), + TestFuncs(vpx_obmc_variance8x8_c, vpx_obmc_variance8x8_sse4_1), + TestFuncs(vpx_obmc_variance8x4_c, vpx_obmc_variance8x4_sse4_1), + TestFuncs(vpx_obmc_variance4x8_c, vpx_obmc_variance4x8_sse4_1), + TestFuncs(vpx_obmc_variance4x4_c, vpx_obmc_variance4x4_sse4_1) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcVarianceTest, + ::testing::ValuesIn(sse4_functions)); +#endif // HAVE_SSE4_1 + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_VP9_HIGHBITDEPTH +class ObmcVarianceHBDTest : public FunctionEquivalenceTest<ObmcVarF> {} + +TEST_P(ObmcVarianceHBDTest, RandomValues) { + DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) { + const int pre_stride = this->rng_(MAX_SB_SIZE + 1); + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + pre[i] = this->rng_(1 << this->bit_depth); + wsrc[i] = this->rng_(1 << this->bit_depth) * + this->rng_(kMaskMax * kMaskMax + 1); + mask[i] = this->rng_(kMaskMax * kMaskMax + 1); + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(pre), pre_stride, + wsrc, mask, &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = tst_func_(CONVERT_TO_BYTEPTR(pre), + pre_stride, wsrc, mask, &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +TEST_P(ObmcVarianceHBDTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) { + const int pre_stride = iter; + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + pre[i] = (1 << this->bit_depth) - 1; + wsrc[i] = ((1 << this->bit_depth) - 1) * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + unsigned int ref_sse, tst_sse; + const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(pre), pre_stride, + wsrc, mask, &ref_sse); + unsigned int tst_res; + ASM_REGISTER_STATE_CHECK( + tst_res = tst_func_(CONVERT_TO_BYTEPTR(pre), pre_stride, + wsrc, mask, &tst_sse)); + + ASSERT_EQ(ref_res, tst_res); + ASSERT_EQ(ref_sse, tst_sse); + } +} + +#if HAVE_SSE4_1 +ObmcVarianceHBDTest::ParamType sse4_functions_hbd[] = { +#if CONFIG_EXT_PARTITION + TestFuncs(vpx_highbd_obmc_variance128x128_c, + vpx_highbd_obmc_variance128x128_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance128x64_c, + vpx_highbd_obmc_variance128x64_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance64x128_c, + vpx_highbd_obmc_variance64x128_sse4_1, 8), +#endif // CONFIG_EXT_PARTITION + TestFuncs(vpx_highbd_obmc_variance64x64_c, + vpx_highbd_obmc_variance64x64_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance64x32_c, + vpx_highbd_obmc_variance64x32_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance32x64_c, + vpx_highbd_obmc_variance32x64_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance32x32_c, + vpx_highbd_obmc_variance32x32_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance32x16_c, + vpx_highbd_obmc_variance32x16_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance16x32_c, + vpx_highbd_obmc_variance16x32_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance16x16_c, + vpx_highbd_obmc_variance16x16_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance16x8_c, + vpx_highbd_obmc_variance16x8_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance8x16_c, + vpx_highbd_obmc_variance8x16_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance8x8_c, + vpx_highbd_obmc_variance8x8_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance8x4_c, + vpx_highbd_obmc_variance8x4_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance4x8_c, + vpx_highbd_obmc_variance4x8_sse4_1, 8), + TestFuncs(vpx_highbd_obmc_variance4x4_c, + vpx_highbd_obmc_variance4x4_sse4_1, 8), +#if CONFIG_EXT_PARTITION + TestFuncs(vpx_highbd_10_obmc_variance128x128_c, + vpx_highbd_10_obmc_variance128x128_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance128x64_c, + vpx_highbd_10_obmc_variance128x64_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance64x128_c, + vpx_highbd_10_obmc_variance64x128_sse4_1, 10), +#endif // CONFIG_EXT_PARTITION + TestFuncs(vpx_highbd_10_obmc_variance64x64_c, + vpx_highbd_10_obmc_variance64x64_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance64x32_c, + vpx_highbd_10_obmc_variance64x32_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance32x64_c, + vpx_highbd_10_obmc_variance32x64_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance32x32_c, + vpx_highbd_10_obmc_variance32x32_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance32x16_c, + vpx_highbd_10_obmc_variance32x16_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance16x32_c, + vpx_highbd_10_obmc_variance16x32_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance16x16_c, + vpx_highbd_10_obmc_variance16x16_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance16x8_c, + vpx_highbd_10_obmc_variance16x8_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance8x16_c, + vpx_highbd_10_obmc_variance8x16_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance8x8_c, + vpx_highbd_10_obmc_variance8x8_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance8x4_c, + vpx_highbd_10_obmc_variance8x4_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance4x8_c, + vpx_highbd_10_obmc_variance4x8_sse4_1, 10), + TestFuncs(vpx_highbd_10_obmc_variance4x4_c, + vpx_highbd_10_obmc_variance4x4_sse4_1, 10), +#if CONFIG_EXT_PARTITION + TestFuncs(vpx_highbd_12_obmc_variance128x128_c, + vpx_highbd_12_obmc_variance128x128_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance128x64_c, + vpx_highbd_12_obmc_variance128x64_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance64x128_c, + vpx_highbd_12_obmc_variance64x128_sse4_1, 12), +#endif // CONFIG_EXT_PARTITION + TestFuncs(vpx_highbd_12_obmc_variance64x64_c, + vpx_highbd_12_obmc_variance64x64_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance64x32_c, + vpx_highbd_12_obmc_variance64x32_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance32x64_c, + vpx_highbd_12_obmc_variance32x64_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance32x32_c, + vpx_highbd_12_obmc_variance32x32_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance32x16_c, + vpx_highbd_12_obmc_variance32x16_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance16x32_c, + vpx_highbd_12_obmc_variance16x32_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance16x16_c, + vpx_highbd_12_obmc_variance16x16_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance16x8_c, + vpx_highbd_12_obmc_variance16x8_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance8x16_c, + vpx_highbd_12_obmc_variance8x16_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance8x8_c, + vpx_highbd_12_obmc_variance8x8_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance8x4_c, + vpx_highbd_12_obmc_variance8x4_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance4x8_c, + vpx_highbd_12_obmc_variance4x8_sse4_1, 12), + TestFuncs(vpx_highbd_12_obmc_variance4x4_c, + vpx_highbd_12_obmc_variance4x4_sse4_1, 12) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcVarianceHBDTest, + ::testing::ValuesIn(sse4_functions_hbd)); +#endif // HAVE_SSE4_1 +#endif // CONFIG_VP9_HIGHBITDEPTH +} // namespace
diff --git a/test/test.mk b/test/test.mk index 67fe705..28c0caa 100644 --- a/test/test.mk +++ b/test/test.mk
@@ -189,6 +189,7 @@ ifeq ($(CONFIG_OBMC),yes) LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += obmc_sad_test.cc +LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += obmc_variance_test.cc endif ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
diff --git a/vpx_dsp/variance.c b/vpx_dsp/variance.c index b9f0e32..af2bba2 100644 --- a/vpx_dsp/variance.c +++ b/vpx_dsp/variance.c
@@ -1025,10 +1025,9 @@ #endif // CONFIG_VP10 && CONFIG_EXT_INTER #if CONFIG_VP10 && CONFIG_OBMC -void obmc_variance(const uint8_t *a, int a_stride, - const int32_t *b, - const int32_t *m, - int w, int h, unsigned int *sse, int *sum) { +static INLINE void obmc_variance(const uint8_t *pre, int pre_stride, + const int32_t *wsrc, const int32_t *mask, + int w, int h, unsigned int *sse, int *sum) { int i, j; *sse = 0; @@ -1036,34 +1035,36 @@ for (i = 0; i < h; i++) { for (j = 0; j < w; j++) { - int diff = ROUND_POWER_OF_TWO_SIGNED(b[j] - a[j] * m[j], 12); + int diff = ROUND_POWER_OF_TWO_SIGNED(wsrc[j] - pre[j] * mask[j], 12); *sum += diff; *sse += diff * diff; } - a += a_stride; - b += w; - m += w; + pre += pre_stride; + wsrc += w; + mask += w; } } #define OBMC_VAR(W, H) \ -unsigned int vpx_obmc_variance##W##x##H##_c(const uint8_t *a, int a_stride, \ - const int32_t *b, \ - const int32_t *m, \ - unsigned int *sse) { \ - int sum; \ - obmc_variance(a, a_stride, b, m, W, H, sse, &sum); \ - return *sse - (((int64_t)sum * sum) / (W * H)); \ +unsigned int vpx_obmc_variance##W##x##H##_c(const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ + unsigned int *sse) { \ + int sum; \ + obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (((int64_t)sum * sum) / (W * H)); \ } #define OBMC_SUBPIX_VAR(W, H) \ -unsigned int vpx_obmc_sub_pixel_variance##W##x##H##_c( \ - const uint8_t *pre, int pre_stride, \ - int xoffset, int yoffset, \ - const int32_t *wsrc, \ - const int32_t *msk, \ - unsigned int *sse) { \ +unsigned int vpx_obmc_sub_pixel_variance##W##x##H##_c(const uint8_t *pre, \ + int pre_stride, \ + int xoffset, \ + int yoffset, \ + const int32_t *wsrc, \ + const int32_t *mask, \ + unsigned int *sse) { \ uint16_t fdata3[(H + 1) * W]; \ uint8_t temp2[H * W]; \ \ @@ -1072,7 +1073,7 @@ var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W, \ bilinear_filters_2t[yoffset]); \ \ - return vpx_obmc_variance##W##x##H##_c(temp2, W, wsrc, msk, sse); \ + return vpx_obmc_variance##W##x##H##_c(temp2, W, wsrc, mask, sse); \ } OBMC_VAR(4, 4) @@ -1126,86 +1127,94 @@ #endif // CONFIG_EXT_PARTITION #if CONFIG_VP9_HIGHBITDEPTH -void highbd_obmc_variance64(const uint8_t *a8, int a_stride, - const int32_t *b, const int32_t *m, - int w, int h, uint64_t *sse, int64_t *sum) { +static INLINE void highbd_obmc_variance64(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + uint64_t *sse, int64_t *sum) { int i, j; - uint16_t *a = CONVERT_TO_SHORTPTR(a8); + uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); *sse = 0; *sum = 0; for (i = 0; i < h; i++) { for (j = 0; j < w; j++) { - int diff = ROUND_POWER_OF_TWO_SIGNED(b[j] - a[j] * m[j], 12); + int diff = ROUND_POWER_OF_TWO_SIGNED(wsrc[j] - pre[j] * mask[j], 12); *sum += diff; *sse += diff * diff; } - a += a_stride; - b += w; - m += w; + pre += pre_stride; + wsrc += w; + mask += w; } } -void highbd_obmc_variance(const uint8_t *a8, int a_stride, - const int32_t *b, const int32_t *m, - int w, int h, unsigned int *sse, int *sum) { +static INLINE void highbd_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + unsigned int *sse, int *sum) { int64_t sum64; uint64_t sse64; - highbd_obmc_variance64(a8, a_stride, b, m, w, h, &sse64, &sum64); + highbd_obmc_variance64(pre8, pre_stride, wsrc, mask, w, h, &sse64, &sum64); *sum = (int)sum64; *sse = (unsigned int)sse64; } -void highbd_10_obmc_variance(const uint8_t *a8, int a_stride, - const int32_t *b, const int32_t *m, - int w, int h, unsigned int *sse, int *sum) { +static INLINE void highbd_10_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + unsigned int *sse, int *sum) { int64_t sum64; uint64_t sse64; - highbd_obmc_variance64(a8, a_stride, b, m, w, h, &sse64, &sum64); + highbd_obmc_variance64(pre8, pre_stride, wsrc, mask, w, h, &sse64, &sum64); *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); } -void highbd_12_obmc_variance(const uint8_t *a8, int a_stride, - const int32_t *b, const int32_t *m, - int w, int h, unsigned int *sse, int *sum) { +static INLINE void highbd_12_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + unsigned int *sse, int *sum) { int64_t sum64; uint64_t sse64; - highbd_obmc_variance64(a8, a_stride, b, m, w, h, &sse64, &sum64); + highbd_obmc_variance64(pre8, pre_stride, wsrc, mask, w, h, &sse64, &sum64); *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); } #define HIGHBD_OBMC_VAR(W, H) \ -unsigned int vpx_highbd_obmc_variance##W##x##H##_c(const uint8_t *a, \ - int a_stride, \ - const int32_t *b, \ - const int32_t *m, \ +unsigned int vpx_highbd_obmc_variance##W##x##H##_c(const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ unsigned int *sse) { \ int sum; \ - highbd_obmc_variance(a, a_stride, b, m, W, H, sse, &sum); \ + highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ return *sse - (((int64_t)sum * sum) / (W * H)); \ } \ \ -unsigned int vpx_highbd_10_obmc_variance##W##x##H##_c(const uint8_t *a, \ - int a_stride, \ - const int32_t *b, \ - const int32_t *m, \ +unsigned int vpx_highbd_10_obmc_variance##W##x##H##_c(const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ unsigned int *sse) { \ int sum; \ - highbd_10_obmc_variance(a, a_stride, b, m, W, H, sse, &sum); \ + highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ return *sse - (((int64_t)sum * sum) / (W * H)); \ } \ \ -unsigned int vpx_highbd_12_obmc_variance##W##x##H##_c(const uint8_t *a, \ - int a_stride, \ - const int32_t *b, \ - const int32_t *m, \ +unsigned int vpx_highbd_12_obmc_variance##W##x##H##_c(const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ unsigned int *sse) { \ int sum; \ - highbd_12_obmc_variance(a, a_stride, b, m, W, H, sse, &sum); \ + highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ return *sse - (((int64_t)sum * sum) / (W * H)); \ } @@ -1214,7 +1223,7 @@ const uint8_t *pre, int pre_stride, \ int xoffset, int yoffset, \ const int32_t *wsrc, \ - const int32_t *msk, \ + const int32_t *mask, \ unsigned int *sse) { \ uint16_t fdata3[(H + 1) * W]; \ uint16_t temp2[H * W]; \ @@ -1226,14 +1235,14 @@ bilinear_filters_2t[yoffset]); \ \ return vpx_highbd_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), \ - W, wsrc, msk, sse); \ + W, wsrc, mask, sse); \ } \ \ unsigned int vpx_highbd_10_obmc_sub_pixel_variance##W##x##H##_c( \ const uint8_t *pre, int pre_stride, \ int xoffset, int yoffset, \ const int32_t *wsrc, \ - const int32_t *msk, \ + const int32_t *mask, \ unsigned int *sse) { \ uint16_t fdata3[(H + 1) * W]; \ uint16_t temp2[H * W]; \ @@ -1245,14 +1254,14 @@ bilinear_filters_2t[yoffset]); \ \ return vpx_highbd_10_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), \ - W, wsrc, msk, sse); \ + W, wsrc, mask, sse); \ } \ \ unsigned int vpx_highbd_12_obmc_sub_pixel_variance##W##x##H##_c( \ const uint8_t *pre, int pre_stride, \ int xoffset, int yoffset, \ const int32_t *wsrc, \ - const int32_t *msk, \ + const int32_t *mask, \ unsigned int *sse) { \ uint16_t fdata3[(H + 1) * W]; \ uint16_t temp2[H * W]; \ @@ -1264,7 +1273,7 @@ bilinear_filters_2t[yoffset]); \ \ return vpx_highbd_12_obmc_variance##W##x##H##_c(CONVERT_TO_BYTEPTR(temp2), \ - W, wsrc, msk, sse); \ + W, wsrc, mask, sse); \ } HIGHBD_OBMC_VAR(4, 4)
diff --git a/vpx_dsp/vpx_dsp.mk b/vpx_dsp/vpx_dsp.mk index 3eb7a9f..d55d952 100644 --- a/vpx_dsp/vpx_dsp.mk +++ b/vpx_dsp/vpx_dsp.mk
@@ -320,6 +320,7 @@ endif #CONFIG_EXT_INTER ifeq ($(CONFIG_OBMC),yes) DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_sad_sse4.c +DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_variance_sse4.c endif #CONFIG_OBMC endif #CONFIG_VP10_ENCODER
diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl index d8055e9..7526bea 100644 --- a/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -1413,9 +1413,9 @@ if (vpx_config("CONFIG_OBMC") eq "yes") { foreach (@block_sizes) { ($w, $h) = @$_; - add_proto qw/unsigned int/, "vpx_obmc_variance${w}x${h}", "const uint8_t *pre_ptr, int pre_stride, const int32_t *wsrc_ptr, const int32_t *mask, unsigned int *sse"; - add_proto qw/unsigned int/, "vpx_obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre_ptr, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc_ptr, const int32_t *mask, unsigned int *sse"; - specialize "vpx_obmc_variance${w}x${h}"; + add_proto qw/unsigned int/, "vpx_obmc_variance${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, unsigned int *sse"; + add_proto qw/unsigned int/, "vpx_obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, unsigned int *sse"; + specialize "vpx_obmc_variance${w}x${h}", q/sse4_1/; specialize "vpx_obmc_sub_pixel_variance${w}x${h}"; } @@ -1423,9 +1423,9 @@ foreach $bd ("_", "_10_", "_12_") { foreach (@block_sizes) { ($w, $h) = @$_; - add_proto qw/unsigned int/, "vpx_highbd${bd}obmc_variance${w}x${h}", "const uint8_t *pre_ptr, int pre_stride, const int32_t *wsrc_ptr, const int32_t *mask, unsigned int *sse"; - add_proto qw/unsigned int/, "vpx_highbd${bd}obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre_ptr, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc_ptr, const int32_t *mask, unsigned int *sse"; - specialize "vpx_highbd${bd}obmc_variance${w}x${h}"; + add_proto qw/unsigned int/, "vpx_highbd${bd}obmc_variance${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, unsigned int *sse"; + add_proto qw/unsigned int/, "vpx_highbd${bd}obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, unsigned int *sse"; + specialize "vpx_highbd${bd}obmc_variance${w}x${h}", qw/sse4_1/; specialize "vpx_highbd${bd}obmc_sub_pixel_variance${w}x${h}"; } }
diff --git a/vpx_dsp/x86/obmc_sad_sse4.c b/vpx_dsp/x86/obmc_sad_sse4.c index de12e1d..e21bb98 100644 --- a/vpx_dsp/x86/obmc_sad_sse4.c +++ b/vpx_dsp/x86/obmc_sad_sse4.c
@@ -15,6 +15,7 @@ #include "vpx_ports/mem.h" #include "vpx/vpx_integer.h" +#include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/x86/synonyms.h" //////////////////////////////////////////////////////////////////////////////// @@ -52,8 +53,7 @@ n += 4; - if (n % 4 == 0) - pre += pre_step; + if (n % 4 == 0) pre += pre_step; } while (n < 4 * height); return xx_hsum_epi32_si32(v_sad_d); @@ -68,7 +68,9 @@ const int pre_step = pre_stride - width; int n = 0; __m128i v_sad_d = _mm_setzero_si128(); - assert(width >= 8 && (width & (width - 1)) == 0); + + assert(width >= 8); + assert(IS_POWER_OF_TWO(width)); do { const __m128i v_p1_b = xx_loadl_32(pre + n + 4); @@ -101,8 +103,7 @@ n += 8; - if (n % width == 0) - pre += pre_step; + if (n % width == 0) pre += pre_step; } while (n < width * height); return xx_hsum_epi32_si32(v_sad_d); @@ -113,10 +114,11 @@ int pre_stride, \ const int32_t *wsrc, \ const int32_t *msk) { \ - if (w == 4) \ + if (w == 4) { \ return obmc_sad_w4(pre, pre_stride, wsrc, msk, h); \ - else \ + } else { \ return obmc_sad_w8n(pre, pre_stride, wsrc, msk, w, h); \ + } \ } #if CONFIG_EXT_PARTITION @@ -175,8 +177,7 @@ n += 4; - if (n % 4 == 0) - pre += pre_step; + if (n % 4 == 0) pre += pre_step; } while (n < 4 * height); return xx_hsum_epi32_si32(v_sad_d); @@ -192,7 +193,9 @@ const int pre_step = pre_stride - width; int n = 0; __m128i v_sad_d = _mm_setzero_si128(); - assert(width >= 8 && (width & (width - 1)) == 0); + + assert(width >= 8); + assert(IS_POWER_OF_TWO(width)); do { const __m128i v_p1_w = xx_loadl_64(pre + n + 4); @@ -225,8 +228,7 @@ n += 8; - if (n % width == 0) - pre += pre_step; + if (n % width == 0) pre += pre_step; } while (n < width * height); return xx_hsum_epi32_si32(v_sad_d); @@ -237,10 +239,11 @@ int pre_stride, \ const int32_t *wsrc, \ const int32_t *mask) { \ - if (w == 4) \ + if (w == 4) { \ return hbd_obmc_sad_w4(pre, pre_stride, wsrc, mask, h); \ - else \ + } else { \ return hbd_obmc_sad_w8n(pre, pre_stride, wsrc, mask, w, h); \ + } \ } #if CONFIG_EXT_PARTITION
diff --git a/vpx_dsp/x86/obmc_variance_sse4.c b/vpx_dsp/x86/obmc_variance_sse4.c new file mode 100644 index 0000000..b967c10 --- /dev/null +++ b/vpx_dsp/x86/obmc_variance_sse4.c
@@ -0,0 +1,379 @@ +/* + * Copyright (c) 2016 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <assert.h> +#include <immintrin.h> + +#include "./vpx_config.h" +#include "vpx_ports/mem.h" +#include "vpx/vpx_integer.h" + +#include "vpx_dsp/vpx_dsp_common.h" +#include "vpx_dsp/x86/synonyms.h" +#include "vpx_dsp/vpx_filter.h" + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +static INLINE void obmc_variance_w4(const uint8_t *pre, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + unsigned int *const sse, + int *const sum, + const int h) { + const int pre_step = pre_stride - 4; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p_b = xx_loadl_32(pre + n); + const __m128i v_m_d = xx_load_128(mask + n); + const __m128i v_w_d = xx_load_128(wsrc + n); + + const __m128i v_p_d = _mm_cvtepu8_epi32(v_p_b); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm_d = _mm_madd_epi16(v_p_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_w_d, v_pm_d); + const __m128i v_rdiff_d = xx_roundn_epi32(v_diff_d, 12); + const __m128i v_sqrdiff_d = _mm_mullo_epi32(v_rdiff_d, v_rdiff_d); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 4; + + if (n % 4 == 0) pre += pre_step; + } while (n < 4 * h); + + *sum = xx_hsum_epi32_si32(v_sum_d); + *sse = xx_hsum_epi32_si32(v_sse_d); +} + +static INLINE void obmc_variance_w8n(const uint8_t *pre, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + unsigned int *const sse, + int *const sum, + const int w, + const int h) { + const int pre_step = pre_stride - w; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(w >= 8); + assert(IS_POWER_OF_TWO(w)); + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p1_b = xx_loadl_32(pre + n + 4); + const __m128i v_m1_d = xx_load_128(mask + n + 4); + const __m128i v_w1_d = xx_load_128(wsrc + n + 4); + const __m128i v_p0_b = xx_loadl_32(pre + n); + const __m128i v_m0_d = xx_load_128(mask + n); + const __m128i v_w0_d = xx_load_128(wsrc + n); + + const __m128i v_p0_d = _mm_cvtepu8_epi32(v_p0_b); + const __m128i v_p1_d = _mm_cvtepu8_epi32(v_p1_b); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm0_d = _mm_madd_epi16(v_p0_d, v_m0_d); + const __m128i v_pm1_d = _mm_madd_epi16(v_p1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_w0_d, v_pm0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_w1_d, v_pm1_d); + + const __m128i v_rdiff0_d = xx_roundn_epi32(v_diff0_d, 12); + const __m128i v_rdiff1_d = xx_roundn_epi32(v_diff1_d, 12); + const __m128i v_rdiff01_w = _mm_packs_epi32(v_rdiff0_d, v_rdiff1_d); + const __m128i v_sqrdiff_d = _mm_madd_epi16(v_rdiff01_w, v_rdiff01_w); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff0_d); + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff1_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 8; + + if (n % w == 0) pre += pre_step; + } while (n < w * h); + + *sum = xx_hsum_epi32_si32(v_sum_d); + *sse = xx_hsum_epi32_si32(v_sse_d); +} + +#define OBMCVARWXH(W, H) \ +unsigned int vpx_obmc_variance##W##x##H##_sse4_1(const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ + unsigned int *sse) { \ + int sum; \ + if (W == 4) { \ + obmc_variance_w4(pre, pre_stride, wsrc, mask, sse, &sum, H); \ + } else { \ + obmc_variance_w8n(pre, pre_stride, wsrc, mask, sse, &sum, W, H); \ + } \ + return *sse - (((int64_t)sum * sum) / (W * H)); \ +} + +#if CONFIG_EXT_PARTITION +OBMCVARWXH(128, 128) +OBMCVARWXH(128, 64) +OBMCVARWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +OBMCVARWXH(64, 64) +OBMCVARWXH(64, 32) +OBMCVARWXH(32, 64) +OBMCVARWXH(32, 32) +OBMCVARWXH(32, 16) +OBMCVARWXH(16, 32) +OBMCVARWXH(16, 16) +OBMCVARWXH(16, 8) +OBMCVARWXH(8, 16) +OBMCVARWXH(8, 8) +OBMCVARWXH(8, 4) +OBMCVARWXH(4, 8) +OBMCVARWXH(4, 4) + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_VP9_HIGHBITDEPTH +static INLINE void hbd_obmc_variance_w4(const uint8_t *pre8, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + uint64_t *const sse, + int64_t *const sum, + const int h) { + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + const int pre_step = pre_stride - 4; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p_w = xx_loadl_64(pre + n); + const __m128i v_m_d = xx_load_128(mask + n); + const __m128i v_w_d = xx_load_128(wsrc + n); + + const __m128i v_p_d = _mm_cvtepu16_epi32(v_p_w); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm_d = _mm_madd_epi16(v_p_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_w_d, v_pm_d); + const __m128i v_rdiff_d = xx_roundn_epi32(v_diff_d, 12); + const __m128i v_sqrdiff_d = _mm_mullo_epi32(v_rdiff_d, v_rdiff_d); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 4; + + if (n % 4 == 0) pre += pre_step; + } while (n < 4 * h); + + *sum = xx_hsum_epi32_si32(v_sum_d); + *sse = xx_hsum_epi32_si32(v_sse_d); +} + +static INLINE void hbd_obmc_variance_w8n(const uint8_t *pre8, + const int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + uint64_t *const sse, + int64_t *const sum, + const int w, + const int h) { + const uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); + const int pre_step = pre_stride - w; + int n = 0; + __m128i v_sum_d = _mm_setzero_si128(); + __m128i v_sse_d = _mm_setzero_si128(); + + assert(w >= 8); + assert(IS_POWER_OF_TWO(w)); + assert(IS_POWER_OF_TWO(h)); + + do { + const __m128i v_p1_w = xx_loadl_64(pre + n + 4); + const __m128i v_m1_d = xx_load_128(mask + n + 4); + const __m128i v_w1_d = xx_load_128(wsrc + n + 4); + const __m128i v_p0_w = xx_loadl_64(pre + n); + const __m128i v_m0_d = xx_load_128(mask + n); + const __m128i v_w0_d = xx_load_128(wsrc + n); + + const __m128i v_p0_d = _mm_cvtepu16_epi32(v_p0_w); + const __m128i v_p1_d = _mm_cvtepu16_epi32(v_p1_w); + + // Values in both pre and mask fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_pm0_d = _mm_madd_epi16(v_p0_d, v_m0_d); + const __m128i v_pm1_d = _mm_madd_epi16(v_p1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_w0_d, v_pm0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_w1_d, v_pm1_d); + + const __m128i v_rdiff0_d = xx_roundn_epi32(v_diff0_d, 12); + const __m128i v_rdiff1_d = xx_roundn_epi32(v_diff1_d, 12); + const __m128i v_rdiff01_w = _mm_packs_epi32(v_rdiff0_d, v_rdiff1_d); + const __m128i v_sqrdiff_d = _mm_madd_epi16(v_rdiff01_w, v_rdiff01_w); + + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff0_d); + v_sum_d = _mm_add_epi32(v_sum_d, v_rdiff1_d); + v_sse_d = _mm_add_epi32(v_sse_d, v_sqrdiff_d); + + n += 8; + + if (n % w == 0) pre += pre_step; + } while (n < w * h); + + *sum += xx_hsum_epi32_si64(v_sum_d); + *sse += xx_hsum_epi32_si64(v_sse_d); +} + +static INLINE void highbd_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64 = 0; + uint64_t sse64 = 0; + if (w == 4) { + hbd_obmc_variance_w4(pre8, pre_stride, wsrc, mask, &sse64, &sum64, h); + } else { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, w, h); + } + *sum = (int)sum64; + *sse = (unsigned int)sse64; +} + +static INLINE void highbd_10_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64 = 0; + uint64_t sse64 = 0; + if (w == 4) { + hbd_obmc_variance_w4(pre8, pre_stride, wsrc, mask, &sse64, &sum64, h); + } else { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, w, h); + } + *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); +} + +static INLINE void highbd_12_obmc_variance(const uint8_t *pre8, int pre_stride, + const int32_t *wsrc, + const int32_t *mask, + int w, int h, + unsigned int *sse, int *sum) { + int64_t sum64 = 0; + uint64_t sse64 = 0; + if (w == 128) { + do { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, + &sse64, &sum64, 128, 32); + pre8 += 32 * pre_stride; + wsrc += 32 * 128; + mask += 32 * 128; + h -= 32; + } while (h > 0); + } else if (w == 64 && h >= 128) { + do { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, + &sse64, &sum64, 64, 64); + pre8 += 64 * pre_stride; + wsrc += 64 * 64; + mask += 64 * 64; + h -= 64; + } while (h > 0); + } else if (w == 4) { + hbd_obmc_variance_w4(pre8, pre_stride, wsrc, mask, &sse64, &sum64, h); + } else { + hbd_obmc_variance_w8n(pre8, pre_stride, wsrc, mask, &sse64, &sum64, w, h); + } + *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); + *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); +} + +#define HBD_OBMCVARWXH(W, H) \ +unsigned int vpx_highbd_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ + unsigned int *sse) { \ + int sum; \ + highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (((int64_t)sum * sum) / (W * H)); \ +} \ + \ +unsigned int vpx_highbd_10_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ + unsigned int *sse) { \ + int sum; \ + highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (((int64_t)sum * sum) / (W * H)); \ +} \ + \ +unsigned int vpx_highbd_12_obmc_variance##W##x##H##_sse4_1( \ + const uint8_t *pre, \ + int pre_stride, \ + const int32_t *wsrc, \ + const int32_t *mask, \ + unsigned int *sse) { \ + int sum; \ + highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \ + return *sse - (((int64_t)sum * sum) / (W * H)); \ +} + +#if CONFIG_EXT_PARTITION +HBD_OBMCVARWXH(128, 128) +HBD_OBMCVARWXH(128, 64) +HBD_OBMCVARWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +HBD_OBMCVARWXH(64, 64) +HBD_OBMCVARWXH(64, 32) +HBD_OBMCVARWXH(32, 64) +HBD_OBMCVARWXH(32, 32) +HBD_OBMCVARWXH(32, 16) +HBD_OBMCVARWXH(16, 32) +HBD_OBMCVARWXH(16, 16) +HBD_OBMCVARWXH(16, 8) +HBD_OBMCVARWXH(8, 16) +HBD_OBMCVARWXH(8, 8) +HBD_OBMCVARWXH(8, 4) +HBD_OBMCVARWXH(4, 8) +HBD_OBMCVARWXH(4, 4) +#endif // CONFIG_VP9_HIGHBITDEPTH
diff --git a/vpx_dsp/x86/synonyms.h b/vpx_dsp/x86/synonyms.h index f1bce0f..6708dd1 100644 --- a/vpx_dsp/x86/synonyms.h +++ b/vpx_dsp/x86/synonyms.h
@@ -72,12 +72,40 @@ return _mm_srli_epi32(v_tmp_d, bits); } +static INLINE __m128i xx_roundn_epi32(__m128i v_val_d, int bits) { + const __m128i v_bias_d = _mm_set1_epi32(1 << (bits - 1)); + const __m128i v_sign_d = _mm_srai_epi32(v_val_d, 31); + const __m128i v_tmp_d = _mm_add_epi32(_mm_add_epi32(v_val_d, v_bias_d), + v_sign_d); + return _mm_srai_epi32(v_tmp_d, bits); +} + #ifdef __SSSE3__ static INLINE int32_t xx_hsum_epi32_si32(__m128i v_d) { v_d = _mm_hadd_epi32(v_d, v_d); v_d = _mm_hadd_epi32(v_d, v_d); return _mm_cvtsi128_si32(v_d); } + +static INLINE int64_t xx_hsum_epi64_si64(__m128i v_q) { + v_q = _mm_add_epi64(v_q, _mm_srli_si128(v_q, 8)); +#if ARCH_X86_64 + return _mm_cvtsi128_si64(v_q); +#else + { + int64_t tmp; + _mm_storel_epi64((__m128i*)&tmp, v_q); + return tmp; + } +#endif +} + +static INLINE int64_t xx_hsum_epi32_si64(__m128i v_d) { + const __m128i v_sign_d = _mm_cmplt_epi32(v_d, _mm_setzero_si128()); + const __m128i v_0_q = _mm_unpacklo_epi32(v_d, v_sign_d); + const __m128i v_1_q = _mm_unpackhi_epi32(v_d, v_sign_d); + return xx_hsum_epi64_si64(_mm_add_epi64(v_0_q, v_1_q)); +} #endif // __SSSE3__ #endif // VPX_DSP_X86_SYNONYS_H_