Add SSE4.1 vpx_obmc_sad* implementations. Speedup for these functions: 4x Change-Id: I21baa04f53c6ab308ea3edf3ebacc62970e97454
diff --git a/test/obmc_sad_test.cc b/test/obmc_sad_test.cc new file mode 100644 index 0000000..9d8c2a2 --- /dev/null +++ b/test/obmc_sad_test.cc
@@ -0,0 +1,200 @@ +/* + * Copyright (c) 2016 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/acm_random.h" + +#include "test/function_equivalence_test.h" + +#include "./vpx_config.h" +#include "./vpx_dsp_rtcd.h" +#include "vpx/vpx_integer.h" + +#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE) + +using std::tr1::make_tuple; + +using libvpx_test::ACMRandom; +using libvpx_test::FunctionEquivalenceTest; + +namespace { + +static const int kIterations = 1000; +static const int kMaskMax = 64; + +typedef unsigned int (*ObmcSadF)(const uint8_t *ref, int ref_stride, + const int32_t *wsrc, const int32_t *mask); + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +class ObmcSadTest : public FunctionEquivalenceTest<ObmcSadF> { + public: + ObmcSadTest() : rng_(ACMRandom::DeterministicSeed()) {} + + protected: + ACMRandom rng_; +}; + +TEST_P(ObmcSadTest, RandomValues) { + DECLARE_ALIGNED(32, uint8_t, ref[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) { + const int ref_stride = rng_(MAX_SB_SIZE + 1); + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + ref[i] = rng_.Rand8(); + wsrc[i] = rng_.Rand8() * rng_(kMaskMax * kMaskMax + 1); + mask[i] = rng_(kMaskMax * kMaskMax + 1); + } + + const unsigned int ref_res = ref_func_(ref, ref_stride, wsrc, mask); + const unsigned int tst_res = tst_func_(ref, ref_stride, wsrc, mask); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(ObmcSadTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint8_t, ref[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) { + const int ref_stride = iter; + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + ref[i] = UINT8_MAX; + wsrc[i] = UINT8_MAX * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + const unsigned int ref_res = ref_func_(ref, ref_stride, wsrc, mask); + const unsigned int tst_res = tst_func_(ref, ref_stride, wsrc, mask); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE4_1 +const ObmcSadTest::ParamType sse4_functions[] = { +#if CONFIG_EXT_PARTITION + make_tuple(vpx_obmc_sad128x128_c, vpx_obmc_sad128x128_sse4_1), + make_tuple(vpx_obmc_sad128x64_c, vpx_obmc_sad128x64_sse4_1), + make_tuple(vpx_obmc_sad64x128_c, vpx_obmc_sad64x128_sse4_1), +#endif // CONFIG_EXT_PARTITION + make_tuple(vpx_obmc_sad64x64_c, vpx_obmc_sad64x64_sse4_1), + make_tuple(vpx_obmc_sad64x32_c, vpx_obmc_sad64x32_sse4_1), + make_tuple(vpx_obmc_sad32x64_c, vpx_obmc_sad32x64_sse4_1), + make_tuple(vpx_obmc_sad32x32_c, vpx_obmc_sad32x32_sse4_1), + make_tuple(vpx_obmc_sad32x16_c, vpx_obmc_sad32x16_sse4_1), + make_tuple(vpx_obmc_sad16x32_c, vpx_obmc_sad16x32_sse4_1), + make_tuple(vpx_obmc_sad16x16_c, vpx_obmc_sad16x16_sse4_1), + make_tuple(vpx_obmc_sad16x8_c, vpx_obmc_sad16x8_sse4_1), + make_tuple(vpx_obmc_sad8x16_c, vpx_obmc_sad8x16_sse4_1), + make_tuple(vpx_obmc_sad8x8_c, vpx_obmc_sad8x8_sse4_1), + make_tuple(vpx_obmc_sad8x4_c, vpx_obmc_sad8x4_sse4_1), + make_tuple(vpx_obmc_sad4x8_c, vpx_obmc_sad4x8_sse4_1), + make_tuple(vpx_obmc_sad4x4_c, vpx_obmc_sad4x4_sse4_1) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcSadTest, + ::testing::ValuesIn(sse4_functions)); +#endif // HAVE_SSE4_1 + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_VP9_HIGHBITDEPTH +class ObmcSadHBDTest : public FunctionEquivalenceTest<ObmcSadF> { + public: + ObmcSadHBDTest() : rng_(ACMRandom::DeterministicSeed()) {} + + protected: + ACMRandom rng_; +}; + +TEST_P(ObmcSadHBDTest, RandomValues) { + DECLARE_ALIGNED(32, uint16_t, ref[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) { + const int ref_stride = rng_(MAX_SB_SIZE + 1); + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + ref[i] = rng_(1<<12); + wsrc[i] = rng_(1<<12) * rng_(kMaskMax * kMaskMax + 1); + mask[i] = rng_(kMaskMax * kMaskMax + 1); + } + + const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(ref), ref_stride, + wsrc, mask); + const unsigned int tst_res = tst_func_(CONVERT_TO_BYTEPTR(ref), ref_stride, + wsrc, mask); + + ASSERT_EQ(ref_res, tst_res); + } +} + +TEST_P(ObmcSadHBDTest, ExtremeValues) { + DECLARE_ALIGNED(32, uint16_t, ref[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]); + DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]); + + for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) { + const int ref_stride = iter; + + for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) { + ref[i] = (1 << 12) - 1; + wsrc[i] = ((1 << 12) - 1) * kMaskMax * kMaskMax; + mask[i] = kMaskMax * kMaskMax; + } + + const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(ref), ref_stride, + wsrc, mask); + const unsigned int tst_res = tst_func_(CONVERT_TO_BYTEPTR(ref), ref_stride, + wsrc, mask); + + ASSERT_EQ(ref_res, tst_res); + } +} + +#if HAVE_SSE4_1 +ObmcSadHBDTest::ParamType sse4_functions_hbd[] = { +#if CONFIG_EXT_PARTITION + make_tuple(vpx_highbd_obmc_sad128x128_c, vpx_highbd_obmc_sad128x128_sse4_1), + make_tuple(vpx_highbd_obmc_sad128x64_c, vpx_highbd_obmc_sad128x64_sse4_1), + make_tuple(vpx_highbd_obmc_sad64x128_c, vpx_highbd_obmc_sad64x128_sse4_1), +#endif // CONFIG_EXT_PARTITION + make_tuple(vpx_highbd_obmc_sad64x64_c, vpx_highbd_obmc_sad64x64_sse4_1), + make_tuple(vpx_highbd_obmc_sad64x32_c, vpx_highbd_obmc_sad64x32_sse4_1), + make_tuple(vpx_highbd_obmc_sad32x64_c, vpx_highbd_obmc_sad32x64_sse4_1), + make_tuple(vpx_highbd_obmc_sad32x32_c, vpx_highbd_obmc_sad32x32_sse4_1), + make_tuple(vpx_highbd_obmc_sad32x16_c, vpx_highbd_obmc_sad32x16_sse4_1), + make_tuple(vpx_highbd_obmc_sad16x32_c, vpx_highbd_obmc_sad16x32_sse4_1), + make_tuple(vpx_highbd_obmc_sad16x16_c, vpx_highbd_obmc_sad16x16_sse4_1), + make_tuple(vpx_highbd_obmc_sad16x8_c, vpx_highbd_obmc_sad16x8_sse4_1), + make_tuple(vpx_highbd_obmc_sad8x16_c, vpx_highbd_obmc_sad8x16_sse4_1), + make_tuple(vpx_highbd_obmc_sad8x8_c, vpx_highbd_obmc_sad8x8_sse4_1), + make_tuple(vpx_highbd_obmc_sad8x4_c, vpx_highbd_obmc_sad8x4_sse4_1), + make_tuple(vpx_highbd_obmc_sad4x8_c, vpx_highbd_obmc_sad4x8_sse4_1), + make_tuple(vpx_highbd_obmc_sad4x4_c, vpx_highbd_obmc_sad4x4_sse4_1) +}; + +INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcSadHBDTest, + ::testing::ValuesIn(sse4_functions_hbd)); +#endif // HAVE_SSE4_1 +#endif // CONFIG_VP9_HIGHBITDEPTH +} // namespace
diff --git a/test/test.mk b/test/test.mk index 0eaee47..fc9d635 100644 --- a/test/test.mk +++ b/test/test.mk
@@ -187,6 +187,10 @@ LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += blend_mask6_test.cc endif +ifeq ($(CONFIG_OBMC),yes) +LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += obmc_sad_test.cc +endif + ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes) LIBVPX_TEST_SRCS-$(HAVE_SSE4_1) += vp10_highbd_iht_test.cc endif # CONFIG_VP9_HIGHBITDEPTH
diff --git a/vpx_dsp/vpx_dsp.mk b/vpx_dsp/vpx_dsp.mk index 7fc06fc..3c519b6 100644 --- a/vpx_dsp/vpx_dsp.mk +++ b/vpx_dsp/vpx_dsp.mk
@@ -314,6 +314,9 @@ DSP_SRCS-$(HAVE_SSSE3) += x86/masked_sad_intrin_ssse3.c DSP_SRCS-$(HAVE_SSSE3) += x86/masked_variance_intrin_ssse3.c endif #CONFIG_EXT_INTER +ifeq ($(CONFIG_OBMC),yes) +DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_sad_sse4.c +endif #CONFIG_OBMC endif #CONFIG_VP10_ENCODER ifeq ($(CONFIG_USE_X86INC),yes)
diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl index 4944b2e..e630994 100644 --- a/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -1104,14 +1104,14 @@ foreach (@block_sizes) { ($w, $h) = @$_; add_proto qw/unsigned int/, "vpx_obmc_sad${w}x${h}", "const uint8_t *ref_ptr, int ref_stride, const int32_t *wsrc_ptr, const int32_t *mask"; - specialize "vpx_obmc_sad${w}x${h}"; + specialize "vpx_obmc_sad${w}x${h}", qw/sse4_1/; } if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { foreach (@block_sizes) { ($w, $h) = @$_; add_proto qw/unsigned int/, "vpx_highbd_obmc_sad${w}x${h}", "const uint8_t *ref_ptr, int ref_stride, const int32_t *wsrc_ptr, const int32_t *mask"; - specialize "vpx_highbd_obmc_sad${w}x${h}"; + specialize "vpx_highbd_obmc_sad${w}x${h}", qw/sse4_1/; } } }
diff --git a/vpx_dsp/x86/obmc_sad_sse4.c b/vpx_dsp/x86/obmc_sad_sse4.c new file mode 100644 index 0000000..57e1428 --- /dev/null +++ b/vpx_dsp/x86/obmc_sad_sse4.c
@@ -0,0 +1,256 @@ +/* + * Copyright (c) 2016 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <assert.h> +#include <immintrin.h> + +#include "./vpx_config.h" +#include "vpx_ports/mem.h" +#include "vpx/vpx_integer.h" + +#include "vpx_dsp/x86/synonyms.h" + +//////////////////////////////////////////////////////////////////////////////// +// 8 bit +//////////////////////////////////////////////////////////////////////////////// + +static INLINE unsigned int obmc_sad_w4(const uint8_t *a, const int a_stride, + const int32_t *b, const int32_t *m, + const int height) { + const int a_step = a_stride - 4; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + + do { + const __m128i v_a_b = xx_loadl_32(a + n); + const __m128i v_m_d = xx_load_128(m + n); + const __m128i v_b_d = xx_load_128(b + n); + + const __m128i v_a_d = _mm_cvtepu8_epi32(v_a_b); + + // Values in both a and m fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_am_d = _mm_madd_epi16(v_a_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_b_d, v_am_d); + const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d); + + // Rounded absolute difference + const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d); + + n += 4; + + if (n % 4 == 0) + a += a_step; + } while (n < 4 * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +static INLINE unsigned int obmc_sad_w8n(const uint8_t *a, const int a_stride, + const int32_t *b, const int32_t *m, + const int width, const int height) { + const int a_step = a_stride - width; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + assert(width >= 8 && (width & (width - 1)) == 0); + + do { + const __m128i v_a1_b = xx_loadl_32(a + n + 4); + const __m128i v_m1_d = xx_load_128(m + n + 4); + const __m128i v_b1_d = xx_load_128(b + n + 4); + const __m128i v_a0_b = xx_loadl_32(a + n); + const __m128i v_m0_d = xx_load_128(m + n); + const __m128i v_b0_d = xx_load_128(b + n); + + const __m128i v_a0_d = _mm_cvtepu8_epi32(v_a0_b); + const __m128i v_a1_d = _mm_cvtepu8_epi32(v_a1_b); + + // Values in both a and m fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_am0_d = _mm_madd_epi16(v_a0_d, v_m0_d); + const __m128i v_am1_d = _mm_madd_epi16(v_a1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_b0_d, v_am0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_b1_d, v_am1_d); + const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d); + const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d); + + // Rounded absolute difference + const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12); + const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d); + v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d); + + n += 8; + + if (n % width == 0) + a += a_step; + } while (n < width * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +#define OBMCSADWXH(w, h) \ +unsigned int vpx_obmc_sad##w##x##h##_sse4_1(const uint8_t *ref, \ + int ref_stride, \ + const int32_t *wsrc, \ + const int32_t *msk) { \ + if (w == 4) \ + return obmc_sad_w4(ref, ref_stride, wsrc, msk, h); \ + else \ + return obmc_sad_w8n(ref, ref_stride, wsrc, msk, w, h); \ +} + +#if CONFIG_EXT_PARTITION +OBMCSADWXH(128, 128) +OBMCSADWXH(128, 64) +OBMCSADWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +OBMCSADWXH(64, 64) +OBMCSADWXH(64, 32) +OBMCSADWXH(32, 64) +OBMCSADWXH(32, 32) +OBMCSADWXH(32, 16) +OBMCSADWXH(16, 32) +OBMCSADWXH(16, 16) +OBMCSADWXH(16, 8) +OBMCSADWXH(8, 16) +OBMCSADWXH(8, 8) +OBMCSADWXH(8, 4) +OBMCSADWXH(4, 8) +OBMCSADWXH(4, 4) + +//////////////////////////////////////////////////////////////////////////////// +// High bit-depth +//////////////////////////////////////////////////////////////////////////////// + +#if CONFIG_VP9_HIGHBITDEPTH +static INLINE unsigned int hbd_obmc_sad_w4(const uint8_t *a8, + const int a_stride, + const int32_t *b, const int32_t *m, + const int height) { + const uint16_t *a = CONVERT_TO_SHORTPTR(a8); + const int a_step = a_stride - 4; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + + do { + const __m128i v_a_w = xx_loadl_64(a + n); + const __m128i v_m_d = xx_load_128(m + n); + const __m128i v_b_d = xx_load_128(b + n); + + const __m128i v_a_d = _mm_cvtepu16_epi32(v_a_w); + + // Values in both a and m fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_am_d = _mm_madd_epi16(v_a_d, v_m_d); + + const __m128i v_diff_d = _mm_sub_epi32(v_b_d, v_am_d); + const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d); + + // Rounded absolute difference + const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d); + + n += 4; + + if (n % 4 == 0) + a += a_step; + } while (n < 4 * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +static INLINE unsigned int hbd_obmc_sad_w8n(const uint8_t *a8, + const int a_stride, + const int32_t *b, const int32_t *m, + const int width, const int height) { + const uint16_t *a = CONVERT_TO_SHORTPTR(a8); + const int a_step = a_stride - width; + int n = 0; + __m128i v_sad_d = _mm_setzero_si128(); + assert(width >= 8 && (width & (width - 1)) == 0); + + do { + const __m128i v_a1_w = xx_loadl_64(a + n + 4); + const __m128i v_m1_d = xx_load_128(m + n + 4); + const __m128i v_b1_d = xx_load_128(b + n + 4); + const __m128i v_a0_w = xx_loadl_64(a + n); + const __m128i v_m0_d = xx_load_128(m + n); + const __m128i v_b0_d = xx_load_128(b + n); + + const __m128i v_a0_d = _mm_cvtepu16_epi32(v_a0_w); + const __m128i v_a1_d = _mm_cvtepu16_epi32(v_a1_w); + + // Values in both a and m fit in 15 bits, and are packed at 32 bit + // boundaries. We use pmaddwd, as it has lower latency on Haswell + // than pmulld but produces the same result with these inputs. + const __m128i v_am0_d = _mm_madd_epi16(v_a0_d, v_m0_d); + const __m128i v_am1_d = _mm_madd_epi16(v_a1_d, v_m1_d); + + const __m128i v_diff0_d = _mm_sub_epi32(v_b0_d, v_am0_d); + const __m128i v_diff1_d = _mm_sub_epi32(v_b1_d, v_am1_d); + const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d); + const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d); + + // Rounded absolute difference + const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12); + const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12); + + v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d); + v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d); + + n += 8; + + if (n % width == 0) + a += a_step; + } while (n < width * height); + + return xx_hsum_epi32_si32(v_sad_d); +} + +#define HBD_OBMCSADWXH(w, h) \ +unsigned int vpx_highbd_obmc_sad##w##x##h##_sse4_1(const uint8_t *ref, \ + int ref_stride, \ + const int32_t *wsrc, \ + const int32_t *msk) { \ + if (w == 4) \ + return hbd_obmc_sad_w4(ref, ref_stride, wsrc, msk, h); \ + else \ + return hbd_obmc_sad_w8n(ref, ref_stride, wsrc, msk, w, h); \ +} + +#if CONFIG_EXT_PARTITION +HBD_OBMCSADWXH(128, 128) +HBD_OBMCSADWXH(128, 64) +HBD_OBMCSADWXH(64, 128) +#endif // CONFIG_EXT_PARTITION +HBD_OBMCSADWXH(64, 64) +HBD_OBMCSADWXH(64, 32) +HBD_OBMCSADWXH(32, 64) +HBD_OBMCSADWXH(32, 32) +HBD_OBMCSADWXH(32, 16) +HBD_OBMCSADWXH(16, 32) +HBD_OBMCSADWXH(16, 16) +HBD_OBMCSADWXH(16, 8) +HBD_OBMCSADWXH(8, 16) +HBD_OBMCSADWXH(8, 8) +HBD_OBMCSADWXH(8, 4) +HBD_OBMCSADWXH(4, 8) +HBD_OBMCSADWXH(4, 4) +#endif // CONFIG_VP9_HIGHBITDEPTH
diff --git a/vpx_dsp/x86/synonyms.h b/vpx_dsp/x86/synonyms.h index 0e7d67d..f1bce0f 100644 --- a/vpx_dsp/x86/synonyms.h +++ b/vpx_dsp/x86/synonyms.h
@@ -66,4 +66,18 @@ return _mm_avg_epu16(v_s_w, _mm_setzero_si128()); } +static INLINE __m128i xx_roundn_epu32(__m128i v_val_d, int bits) { + const __m128i v_bias_d = _mm_set1_epi32(1 << (bits - 1)); + const __m128i v_tmp_d = _mm_add_epi32(v_val_d, v_bias_d); + return _mm_srli_epi32(v_tmp_d, bits); +} + +#ifdef __SSSE3__ +static INLINE int32_t xx_hsum_epi32_si32(__m128i v_d) { + v_d = _mm_hadd_epi32(v_d, v_d); + v_d = _mm_hadd_epi32(v_d, v_d); + return _mm_cvtsi128_si32(v_d); +} +#endif // __SSSE3__ + #endif // VPX_DSP_X86_SYNONYS_H_