Highbd intra pred H_PRED sse2 optimization
sse2 v. C speedup:
4x4 ~8.0x
8x8 ~8.2x
16x16 ~6.5x
32x32 ~3.8x
Blocksize:
4x4, 4x8, 8x4, 8x8, 8x16, 16x8, 16x16, 16x32, 32x16, 32x32
Square blocksize code is from libvpx:
"30d9a1916 vpxdsp: [x86] add highbd_h_predictor functions",
Credit goes to Scott LaVarnway. Speed tests do not support
rectangular blocksize yet.
Change-Id: I9a1f24aecab8de94f8ea59ec8748fe3537d721ae
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake
index b7f9b6b..0da4392 100644
--- a/aom_dsp/aom_dsp.cmake
+++ b/aom_dsp/aom_dsp.cmake
@@ -217,6 +217,7 @@
set(AOM_DSP_COMMON_INTRIN_SSE2
${AOM_DSP_COMMON_INTRIN_SSE2}
+ "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.c"
"${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c")
set(AOM_DSP_COMMON_INTRIN_AVX2
diff --git a/aom_dsp/aom_dsp.mk b/aom_dsp/aom_dsp.mk
index 3eb0e3b..aebeef7 100644
--- a/aom_dsp/aom_dsp.mk
+++ b/aom_dsp/aom_dsp.mk
@@ -78,6 +78,7 @@
ifeq ($(CONFIG_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_SSE) += x86/highbd_intrapred_sse2.asm
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.asm
+DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.c
endif # CONFIG_HIGHBITDEPTH
DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM)
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 8400584..ed794fd 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -123,6 +123,16 @@
specialize qw/aom_highbd_dc_predictor_16x16 sse2/;
specialize qw/aom_highbd_v_predictor_32x32 sse2/;
specialize qw/aom_highbd_dc_predictor_32x32 sse2/;
+ specialize qw/aom_highbd_h_predictor_4x4 sse2/;
+ specialize qw/aom_highbd_h_predictor_4x8 sse2/;
+ specialize qw/aom_highbd_h_predictor_8x4 sse2/;
+ specialize qw/aom_highbd_h_predictor_8x8 sse2/;
+ specialize qw/aom_highbd_h_predictor_8x16 sse2/;
+ specialize qw/aom_highbd_h_predictor_16x8 sse2/;
+ specialize qw/aom_highbd_h_predictor_16x16 sse2/;
+ specialize qw/aom_highbd_h_predictor_16x32 sse2/;
+ specialize qw/aom_highbd_h_predictor_32x16 sse2/;
+ specialize qw/aom_highbd_h_predictor_32x32 sse2/;
} # CONFIG_HIGHBITDEPTH
#
diff --git a/aom_dsp/x86/highbd_intrapred_sse2.c b/aom_dsp/x86/highbd_intrapred_sse2.c
new file mode 100644
index 0000000..a2ae293
--- /dev/null
+++ b/aom_dsp/x86/highbd_intrapred_sse2.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2017, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <emmintrin.h>
+
+#include "./aom_dsp_rtcd.h"
+
+// -----------------------------------------------------------------------------
+// H_PRED
+
+void aom_highbd_h_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ const __m128i left_u16 = _mm_loadl_epi64((const __m128i *)left);
+ const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0);
+ const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55);
+ const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa);
+ const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff);
+ (void)above;
+ (void)bd;
+ _mm_storel_epi64((__m128i *)dst, row0);
+ dst += stride;
+ _mm_storel_epi64((__m128i *)dst, row1);
+ dst += stride;
+ _mm_storel_epi64((__m128i *)dst, row2);
+ dst += stride;
+ _mm_storel_epi64((__m128i *)dst, row3);
+}
+
+void aom_highbd_h_predictor_4x8_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ aom_highbd_h_predictor_4x4_sse2(dst, stride, above, left, bd);
+ dst += stride << 2;
+ left += 4;
+ aom_highbd_h_predictor_4x4_sse2(dst, stride, above, left, bd);
+}
+
+void aom_highbd_h_predictor_8x4_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ const __m128i left_u16 = _mm_load_si128((const __m128i *)left);
+ const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0);
+ const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55);
+ const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa);
+ const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff);
+ (void)above;
+ (void)bd;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row0, row0));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row1, row1));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row2, row2));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row3, row3));
+}
+
+void aom_highbd_h_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ const __m128i left_u16 = _mm_load_si128((const __m128i *)left);
+ const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0);
+ const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55);
+ const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa);
+ const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff);
+ const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0);
+ const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55);
+ const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa);
+ const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff);
+ (void)above;
+ (void)bd;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row0, row0));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row1, row1));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row2, row2));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row3, row3));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row4, row4));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row5, row5));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row6, row6));
+ dst += stride;
+ _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row7, row7));
+}
+
+void aom_highbd_h_predictor_8x16_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ aom_highbd_h_predictor_8x8_sse2(dst, stride, above, left, bd);
+ dst += stride << 3;
+ left += 8;
+ aom_highbd_h_predictor_8x8_sse2(dst, stride, above, left, bd);
+}
+
+static INLINE void h_store_16_unpacklo(uint16_t **dst, const ptrdiff_t stride,
+ const __m128i *row) {
+ const __m128i val = _mm_unpacklo_epi64(*row, *row);
+ _mm_store_si128((__m128i *)*dst, val);
+ _mm_store_si128((__m128i *)(*dst + 8), val);
+ *dst += stride;
+}
+
+static INLINE void h_store_16_unpackhi(uint16_t **dst, const ptrdiff_t stride,
+ const __m128i *row) {
+ const __m128i val = _mm_unpackhi_epi64(*row, *row);
+ _mm_store_si128((__m128i *)(*dst), val);
+ _mm_store_si128((__m128i *)(*dst + 8), val);
+ *dst += stride;
+}
+
+static INLINE void h_predictor_16x8(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *left) {
+ const __m128i left_u16 = _mm_load_si128((const __m128i *)left);
+ const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0);
+ const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55);
+ const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa);
+ const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff);
+ const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0);
+ const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55);
+ const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa);
+ const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff);
+ h_store_16_unpacklo(&dst, stride, &row0);
+ h_store_16_unpacklo(&dst, stride, &row1);
+ h_store_16_unpacklo(&dst, stride, &row2);
+ h_store_16_unpacklo(&dst, stride, &row3);
+ h_store_16_unpackhi(&dst, stride, &row4);
+ h_store_16_unpackhi(&dst, stride, &row5);
+ h_store_16_unpackhi(&dst, stride, &row6);
+ h_store_16_unpackhi(&dst, stride, &row7);
+}
+
+void aom_highbd_h_predictor_16x8_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ (void)above;
+ (void)bd;
+ h_predictor_16x8(dst, stride, left);
+}
+
+void aom_highbd_h_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ int i;
+ (void)above;
+ (void)bd;
+
+ for (i = 0; i < 2; i++, left += 8) {
+ h_predictor_16x8(dst, stride, left);
+ dst += stride << 3;
+ }
+}
+
+void aom_highbd_h_predictor_16x32_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ int i;
+ (void)above;
+ (void)bd;
+
+ for (i = 0; i < 4; i++, left += 8) {
+ h_predictor_16x8(dst, stride, left);
+ dst += stride << 3;
+ }
+}
+
+static INLINE void h_store_32_unpacklo(uint16_t **dst, const ptrdiff_t stride,
+ const __m128i *row) {
+ const __m128i val = _mm_unpacklo_epi64(*row, *row);
+ _mm_store_si128((__m128i *)(*dst), val);
+ _mm_store_si128((__m128i *)(*dst + 8), val);
+ _mm_store_si128((__m128i *)(*dst + 16), val);
+ _mm_store_si128((__m128i *)(*dst + 24), val);
+ *dst += stride;
+}
+
+static INLINE void h_store_32_unpackhi(uint16_t **dst, const ptrdiff_t stride,
+ const __m128i *row) {
+ const __m128i val = _mm_unpackhi_epi64(*row, *row);
+ _mm_store_si128((__m128i *)(*dst), val);
+ _mm_store_si128((__m128i *)(*dst + 8), val);
+ _mm_store_si128((__m128i *)(*dst + 16), val);
+ _mm_store_si128((__m128i *)(*dst + 24), val);
+ *dst += stride;
+}
+
+static INLINE void h_predictor_32x8(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *left) {
+ const __m128i left_u16 = _mm_load_si128((const __m128i *)left);
+ const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0);
+ const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55);
+ const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa);
+ const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff);
+ const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0);
+ const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55);
+ const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa);
+ const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff);
+ h_store_32_unpacklo(&dst, stride, &row0);
+ h_store_32_unpacklo(&dst, stride, &row1);
+ h_store_32_unpacklo(&dst, stride, &row2);
+ h_store_32_unpacklo(&dst, stride, &row3);
+ h_store_32_unpackhi(&dst, stride, &row4);
+ h_store_32_unpackhi(&dst, stride, &row5);
+ h_store_32_unpackhi(&dst, stride, &row6);
+ h_store_32_unpackhi(&dst, stride, &row7);
+}
+
+void aom_highbd_h_predictor_32x16_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ int i;
+ (void)above;
+ (void)bd;
+
+ for (i = 0; i < 2; i++, left += 8) {
+ h_predictor_32x8(dst, stride, left);
+ dst += stride << 3;
+ }
+}
+
+void aom_highbd_h_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride,
+ const uint16_t *above,
+ const uint16_t *left, int bd) {
+ int i;
+ (void)above;
+ (void)bd;
+
+ for (i = 0; i < 4; i++, left += 8) {
+ h_predictor_32x8(dst, stride, left);
+ dst += stride << 3;
+ }
+}
diff --git a/test/intrapred_test.cc b/test/intrapred_test.cc
index e8e87c4..06b1e95 100644
--- a/test/intrapred_test.cc
+++ b/test/intrapred_test.cc
@@ -140,6 +140,11 @@
highbd_entry(dc, 16, 16, sse2, 8), highbd_entry(dc, 32, 32, sse2, 8),
highbd_entry(v, 4, 4, sse2, 8), highbd_entry(v, 8, 8, sse2, 8),
highbd_entry(v, 16, 16, sse2, 8), highbd_entry(v, 32, 32, sse2, 8),
+ highbd_entry(h, 4, 4, sse2, 8), highbd_entry(h, 4, 8, sse2, 8),
+ highbd_entry(h, 8, 4, sse2, 8), highbd_entry(h, 8, 8, sse2, 8),
+ highbd_entry(h, 8, 16, sse2, 8), highbd_entry(h, 16, 8, sse2, 8),
+ highbd_entry(h, 16, 16, sse2, 8), highbd_entry(h, 16, 32, sse2, 8),
+ highbd_entry(h, 32, 16, sse2, 8), highbd_entry(h, 32, 32, sse2, 8),
};
INSTANTIATE_TEST_CASE_P(SSE2_TO_C_8, AV1IntraPredTest,
@@ -150,6 +155,11 @@
highbd_entry(dc, 16, 16, sse2, 10), highbd_entry(dc, 32, 32, sse2, 10),
highbd_entry(v, 4, 4, sse2, 10), highbd_entry(v, 8, 8, sse2, 10),
highbd_entry(v, 16, 16, sse2, 10), highbd_entry(v, 32, 32, sse2, 10),
+ highbd_entry(h, 4, 4, sse2, 10), highbd_entry(h, 4, 8, sse2, 10),
+ highbd_entry(h, 8, 4, sse2, 10), highbd_entry(h, 8, 8, sse2, 10),
+ highbd_entry(h, 8, 16, sse2, 10), highbd_entry(h, 16, 8, sse2, 10),
+ highbd_entry(h, 16, 16, sse2, 10), highbd_entry(h, 16, 32, sse2, 10),
+ highbd_entry(h, 32, 16, sse2, 10), highbd_entry(h, 32, 32, sse2, 10),
};
INSTANTIATE_TEST_CASE_P(SSE2_TO_C_10, AV1IntraPredTest,
@@ -160,6 +170,11 @@
highbd_entry(dc, 16, 16, sse2, 12), highbd_entry(dc, 32, 32, sse2, 12),
highbd_entry(v, 4, 4, sse2, 12), highbd_entry(v, 8, 8, sse2, 12),
highbd_entry(v, 16, 16, sse2, 12), highbd_entry(v, 32, 32, sse2, 12),
+ highbd_entry(h, 4, 4, sse2, 12), highbd_entry(h, 4, 8, sse2, 12),
+ highbd_entry(h, 8, 4, sse2, 12), highbd_entry(h, 8, 8, sse2, 12),
+ highbd_entry(h, 8, 16, sse2, 12), highbd_entry(h, 16, 8, sse2, 12),
+ highbd_entry(h, 16, 16, sse2, 12), highbd_entry(h, 16, 32, sse2, 12),
+ highbd_entry(h, 32, 16, sse2, 12), highbd_entry(h, 32, 32, sse2, 12),
};
INSTANTIATE_TEST_CASE_P(SSE2_TO_C_12, AV1IntraPredTest,
diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc
index 73d1745..9525a62 100644
--- a/test/test_intra_pred_speed.cc
+++ b/test/test_intra_pred_speed.cc
@@ -76,7 +76,7 @@
int idx) {
libaom_test::MD5 md5;
md5.Add(reinterpret_cast<const uint8_t *>(data), data_size);
- printf("Mode %s[%12s]: %5d ms MD5: %s\n", name, kAv1IntraPredNames[idx],
+ printf("Mode %s[%13s]: %5d ms MD5: %s\n", name, kAv1IntraPredNames[idx],
elapsed_time, md5.Get());
EXPECT_STREQ(signatures[idx], md5.Get());
}
@@ -634,8 +634,8 @@
#if HAVE_SSE2
HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, NULL, NULL, NULL, NULL, NULL,
- NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
- NULL, NULL)
+ aom_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL,
+ NULL, NULL, NULL, NULL, NULL, NULL)
#endif
// -----------------------------------------------------------------------------
@@ -663,8 +663,8 @@
#if HAVE_SSE2
HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, NULL, NULL, NULL, NULL, NULL,
- NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
- NULL, NULL)
+ aom_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL,
+ NULL, NULL, NULL, NULL, NULL, NULL)
#endif
// -----------------------------------------------------------------------------
@@ -692,8 +692,8 @@
#if HAVE_SSE2
HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, NULL, NULL, NULL, NULL,
- NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
- NULL, NULL, NULL)
+ NULL, aom_highbd_h_predictor_16x16_sse2, NULL, NULL,
+ NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
#endif
// -----------------------------------------------------------------------------
@@ -721,8 +721,8 @@
#if HAVE_SSE2
HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, NULL, NULL, NULL, NULL,
- NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
- NULL, NULL, NULL)
+ NULL, aom_highbd_h_predictor_32x32_sse2, NULL, NULL,
+ NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
#endif
#endif // CONFIG_HIGHBITDEPTH
