Highbd intra pred H_PRED sse2 optimization sse2 v. C speedup: 4x4 ~8.0x 8x8 ~8.2x 16x16 ~6.5x 32x32 ~3.8x Blocksize: 4x4, 4x8, 8x4, 8x8, 8x16, 16x8, 16x16, 16x32, 32x16, 32x32 Square blocksize code is from libvpx: "30d9a1916 vpxdsp: [x86] add highbd_h_predictor functions", Credit goes to Scott LaVarnway. Speed tests do not support rectangular blocksize yet. Change-Id: I9a1f24aecab8de94f8ea59ec8748fe3537d721ae
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index b7f9b6b..0da4392 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake
@@ -217,6 +217,7 @@ set(AOM_DSP_COMMON_INTRIN_SSE2 ${AOM_DSP_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.c" "${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c") set(AOM_DSP_COMMON_INTRIN_AVX2
diff --git a/aom_dsp/aom_dsp.mk b/aom_dsp/aom_dsp.mk index 3eb0e3b..aebeef7 100644 --- a/aom_dsp/aom_dsp.mk +++ b/aom_dsp/aom_dsp.mk
@@ -78,6 +78,7 @@ ifeq ($(CONFIG_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_SSE) += x86/highbd_intrapred_sse2.asm DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.c endif # CONFIG_HIGHBITDEPTH DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM)
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 8400584..ed794fd 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -123,6 +123,16 @@ specialize qw/aom_highbd_dc_predictor_16x16 sse2/; specialize qw/aom_highbd_v_predictor_32x32 sse2/; specialize qw/aom_highbd_dc_predictor_32x32 sse2/; + specialize qw/aom_highbd_h_predictor_4x4 sse2/; + specialize qw/aom_highbd_h_predictor_4x8 sse2/; + specialize qw/aom_highbd_h_predictor_8x4 sse2/; + specialize qw/aom_highbd_h_predictor_8x8 sse2/; + specialize qw/aom_highbd_h_predictor_8x16 sse2/; + specialize qw/aom_highbd_h_predictor_16x8 sse2/; + specialize qw/aom_highbd_h_predictor_16x16 sse2/; + specialize qw/aom_highbd_h_predictor_16x32 sse2/; + specialize qw/aom_highbd_h_predictor_32x16 sse2/; + specialize qw/aom_highbd_h_predictor_32x32 sse2/; } # CONFIG_HIGHBITDEPTH #
diff --git a/aom_dsp/x86/highbd_intrapred_sse2.c b/aom_dsp/x86/highbd_intrapred_sse2.c new file mode 100644 index 0000000..a2ae293 --- /dev/null +++ b/aom_dsp/x86/highbd_intrapred_sse2.c
@@ -0,0 +1,242 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <emmintrin.h> + +#include "./aom_dsp_rtcd.h" + +// ----------------------------------------------------------------------------- +// H_PRED + +void aom_highbd_h_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i left_u16 = _mm_loadl_epi64((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + (void)above; + (void)bd; + _mm_storel_epi64((__m128i *)dst, row0); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row1); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row2); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row3); +} + +void aom_highbd_h_predictor_4x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + aom_highbd_h_predictor_4x4_sse2(dst, stride, above, left, bd); + dst += stride << 2; + left += 4; + aom_highbd_h_predictor_4x4_sse2(dst, stride, above, left, bd); +} + +void aom_highbd_h_predictor_8x4_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + (void)above; + (void)bd; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row0, row0)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row1, row1)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row2, row2)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row3, row3)); +} + +void aom_highbd_h_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0); + const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55); + const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa); + const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff); + (void)above; + (void)bd; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row0, row0)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row1, row1)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row2, row2)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row3, row3)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row4, row4)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row5, row5)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row6, row6)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row7, row7)); +} + +void aom_highbd_h_predictor_8x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + aom_highbd_h_predictor_8x8_sse2(dst, stride, above, left, bd); + dst += stride << 3; + left += 8; + aom_highbd_h_predictor_8x8_sse2(dst, stride, above, left, bd); +} + +static INLINE void h_store_16_unpacklo(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpacklo_epi64(*row, *row); + _mm_store_si128((__m128i *)*dst, val); + _mm_store_si128((__m128i *)(*dst + 8), val); + *dst += stride; +} + +static INLINE void h_store_16_unpackhi(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpackhi_epi64(*row, *row); + _mm_store_si128((__m128i *)(*dst), val); + _mm_store_si128((__m128i *)(*dst + 8), val); + *dst += stride; +} + +static INLINE void h_predictor_16x8(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0); + const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55); + const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa); + const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff); + h_store_16_unpacklo(&dst, stride, &row0); + h_store_16_unpacklo(&dst, stride, &row1); + h_store_16_unpacklo(&dst, stride, &row2); + h_store_16_unpacklo(&dst, stride, &row3); + h_store_16_unpackhi(&dst, stride, &row4); + h_store_16_unpackhi(&dst, stride, &row5); + h_store_16_unpackhi(&dst, stride, &row6); + h_store_16_unpackhi(&dst, stride, &row7); +} + +void aom_highbd_h_predictor_16x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + (void)above; + (void)bd; + h_predictor_16x8(dst, stride, left); +} + +void aom_highbd_h_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 2; i++, left += 8) { + h_predictor_16x8(dst, stride, left); + dst += stride << 3; + } +} + +void aom_highbd_h_predictor_16x32_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 4; i++, left += 8) { + h_predictor_16x8(dst, stride, left); + dst += stride << 3; + } +} + +static INLINE void h_store_32_unpacklo(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpacklo_epi64(*row, *row); + _mm_store_si128((__m128i *)(*dst), val); + _mm_store_si128((__m128i *)(*dst + 8), val); + _mm_store_si128((__m128i *)(*dst + 16), val); + _mm_store_si128((__m128i *)(*dst + 24), val); + *dst += stride; +} + +static INLINE void h_store_32_unpackhi(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpackhi_epi64(*row, *row); + _mm_store_si128((__m128i *)(*dst), val); + _mm_store_si128((__m128i *)(*dst + 8), val); + _mm_store_si128((__m128i *)(*dst + 16), val); + _mm_store_si128((__m128i *)(*dst + 24), val); + *dst += stride; +} + +static INLINE void h_predictor_32x8(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0); + const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55); + const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa); + const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff); + h_store_32_unpacklo(&dst, stride, &row0); + h_store_32_unpacklo(&dst, stride, &row1); + h_store_32_unpacklo(&dst, stride, &row2); + h_store_32_unpacklo(&dst, stride, &row3); + h_store_32_unpackhi(&dst, stride, &row4); + h_store_32_unpackhi(&dst, stride, &row5); + h_store_32_unpackhi(&dst, stride, &row6); + h_store_32_unpackhi(&dst, stride, &row7); +} + +void aom_highbd_h_predictor_32x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 2; i++, left += 8) { + h_predictor_32x8(dst, stride, left); + dst += stride << 3; + } +} + +void aom_highbd_h_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 4; i++, left += 8) { + h_predictor_32x8(dst, stride, left); + dst += stride << 3; + } +}
diff --git a/test/intrapred_test.cc b/test/intrapred_test.cc index e8e87c4..06b1e95 100644 --- a/test/intrapred_test.cc +++ b/test/intrapred_test.cc
@@ -140,6 +140,11 @@ highbd_entry(dc, 16, 16, sse2, 8), highbd_entry(dc, 32, 32, sse2, 8), highbd_entry(v, 4, 4, sse2, 8), highbd_entry(v, 8, 8, sse2, 8), highbd_entry(v, 16, 16, sse2, 8), highbd_entry(v, 32, 32, sse2, 8), + highbd_entry(h, 4, 4, sse2, 8), highbd_entry(h, 4, 8, sse2, 8), + highbd_entry(h, 8, 4, sse2, 8), highbd_entry(h, 8, 8, sse2, 8), + highbd_entry(h, 8, 16, sse2, 8), highbd_entry(h, 16, 8, sse2, 8), + highbd_entry(h, 16, 16, sse2, 8), highbd_entry(h, 16, 32, sse2, 8), + highbd_entry(h, 32, 16, sse2, 8), highbd_entry(h, 32, 32, sse2, 8), }; INSTANTIATE_TEST_CASE_P(SSE2_TO_C_8, AV1IntraPredTest, @@ -150,6 +155,11 @@ highbd_entry(dc, 16, 16, sse2, 10), highbd_entry(dc, 32, 32, sse2, 10), highbd_entry(v, 4, 4, sse2, 10), highbd_entry(v, 8, 8, sse2, 10), highbd_entry(v, 16, 16, sse2, 10), highbd_entry(v, 32, 32, sse2, 10), + highbd_entry(h, 4, 4, sse2, 10), highbd_entry(h, 4, 8, sse2, 10), + highbd_entry(h, 8, 4, sse2, 10), highbd_entry(h, 8, 8, sse2, 10), + highbd_entry(h, 8, 16, sse2, 10), highbd_entry(h, 16, 8, sse2, 10), + highbd_entry(h, 16, 16, sse2, 10), highbd_entry(h, 16, 32, sse2, 10), + highbd_entry(h, 32, 16, sse2, 10), highbd_entry(h, 32, 32, sse2, 10), }; INSTANTIATE_TEST_CASE_P(SSE2_TO_C_10, AV1IntraPredTest, @@ -160,6 +170,11 @@ highbd_entry(dc, 16, 16, sse2, 12), highbd_entry(dc, 32, 32, sse2, 12), highbd_entry(v, 4, 4, sse2, 12), highbd_entry(v, 8, 8, sse2, 12), highbd_entry(v, 16, 16, sse2, 12), highbd_entry(v, 32, 32, sse2, 12), + highbd_entry(h, 4, 4, sse2, 12), highbd_entry(h, 4, 8, sse2, 12), + highbd_entry(h, 8, 4, sse2, 12), highbd_entry(h, 8, 8, sse2, 12), + highbd_entry(h, 8, 16, sse2, 12), highbd_entry(h, 16, 8, sse2, 12), + highbd_entry(h, 16, 16, sse2, 12), highbd_entry(h, 16, 32, sse2, 12), + highbd_entry(h, 32, 16, sse2, 12), highbd_entry(h, 32, 32, sse2, 12), }; INSTANTIATE_TEST_CASE_P(SSE2_TO_C_12, AV1IntraPredTest,
diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc index 73d1745..9525a62 100644 --- a/test/test_intra_pred_speed.cc +++ b/test/test_intra_pred_speed.cc
@@ -76,7 +76,7 @@ int idx) { libaom_test::MD5 md5; md5.Add(reinterpret_cast<const uint8_t *>(data), data_size); - printf("Mode %s[%12s]: %5d ms MD5: %s\n", name, kAv1IntraPredNames[idx], + printf("Mode %s[%13s]: %5d ms MD5: %s\n", name, kAv1IntraPredNames[idx], elapsed_time, md5.Get()); EXPECT_STREQ(signatures[idx], md5.Get()); } @@ -634,8 +634,8 @@ #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL) + aom_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL) #endif // ----------------------------------------------------------------------------- @@ -663,8 +663,8 @@ #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL) + aom_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL) #endif // ----------------------------------------------------------------------------- @@ -692,8 +692,8 @@ #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL) + NULL, aom_highbd_h_predictor_16x16_sse2, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) #endif // ----------------------------------------------------------------------------- @@ -721,8 +721,8 @@ #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL) + NULL, aom_highbd_h_predictor_32x32_sse2, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) #endif #endif // CONFIG_HIGHBITDEPTH