Add sse2/avx2 version of aom_dc_left_predictor_wxh() for width or height equal to 64. Change-Id: I911ee212484ff68163f906f7e3317f093451002a
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 421524c..9437e13 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -102,6 +102,10 @@ specialize qw/aom_dc_left_predictor_16x32 sse2/; specialize qw/aom_dc_left_predictor_32x16 sse2 avx2/; specialize qw/aom_dc_left_predictor_32x32 msa neon sse2 avx2/; +specialize qw/aom_dc_left_predictor_32x64 sse2 avx2/; +specialize qw/aom_dc_left_predictor_64x64 sse2 avx2/; +specialize qw/aom_dc_left_predictor_64x32 sse2 avx2/; +specialize qw/aom_dc_left_predictor_64x16 sse2 avx2/; specialize qw/aom_dc_128_predictor_4x4 msa neon sse2/; specialize qw/aom_dc_128_predictor_4x8 sse2/; specialize qw/aom_dc_128_predictor_8x4 sse2/;
diff --git a/aom_dsp/x86/intrapred_avx2.c b/aom_dsp/x86/intrapred_avx2.c index 8ebf3c9..c3d6829 100644 --- a/aom_dsp/x86/intrapred_avx2.c +++ b/aom_dsp/x86/intrapred_avx2.c
@@ -13,6 +13,19 @@ #include "./aom_dsp_rtcd.h" +static INLINE __m256i dc_sum_64(const uint8_t *ref) { + const __m256i x0 = _mm256_loadu_si256((const __m256i *)ref); + const __m256i x1 = _mm256_loadu_si256((const __m256i *)(ref + 32)); + const __m256i zero = _mm256_setzero_si256(); + __m256i y0 = _mm256_sad_epu8(x0, zero); + __m256i y1 = _mm256_sad_epu8(x1, zero); + y0 = _mm256_add_epi64(y0, y1); + __m256i u0 = _mm256_permute2x128_si256(y0, y0, 1); + y0 = _mm256_add_epi64(u0, y0); + u0 = _mm256_unpackhi_epi64(y0, y0); + return _mm256_add_epi16(y0, u0); +} + static INLINE __m256i dc_sum_32(const uint8_t *ref) { const __m256i x = _mm256_loadu_si256((const __m256i *)ref); const __m256i zero = _mm256_setzero_si256(); @@ -25,8 +38,7 @@ static INLINE void row_store_32xh(const __m256i *r, int height, uint8_t *dst, ptrdiff_t stride) { - int i; - for (i = 0; i < height; ++i) { + for (int i = 0; i < height; ++i) { _mm256_storeu_si256((__m256i *)dst, *r); dst += stride; } @@ -34,8 +46,7 @@ static INLINE void row_store_64xh(const __m256i *r, int height, uint8_t *dst, ptrdiff_t stride) { - int i; - for (i = 0; i < height; ++i) { + for (int i = 0; i < height; ++i) { _mm256_storeu_si256((__m256i *)dst, *r); _mm256_storeu_si256((__m256i *)(dst + 32), *r); dst += stride; @@ -212,6 +223,63 @@ row_store_32xh(&row, 16, dst, stride); } +void aom_dc_left_predictor_32x64_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + __m256i sum = dc_sum_64(left); + (void)above; + + const __m256i thirtytwo = _mm256_set1_epi16(32); + sum = _mm256_add_epi16(sum, thirtytwo); + sum = _mm256_srai_epi16(sum, 6); + const __m256i zero = _mm256_setzero_si256(); + __m256i row = _mm256_shuffle_epi8(sum, zero); + row_store_32xh(&row, 64, dst, stride); +} + +void aom_dc_left_predictor_64x64_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + __m256i sum = dc_sum_64(left); + (void)above; + + const __m256i thirtytwo = _mm256_set1_epi16(32); + sum = _mm256_add_epi16(sum, thirtytwo); + sum = _mm256_srai_epi16(sum, 6); + const __m256i zero = _mm256_setzero_si256(); + __m256i row = _mm256_shuffle_epi8(sum, zero); + row_store_64xh(&row, 64, dst, stride); +} + +void aom_dc_left_predictor_64x32_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + __m256i sum = dc_sum_32(left); + (void)above; + + const __m256i sixteen = _mm256_set1_epi16(16); + sum = _mm256_add_epi16(sum, sixteen); + sum = _mm256_srai_epi16(sum, 5); + const __m256i zero = _mm256_setzero_si256(); + __m256i row = _mm256_shuffle_epi8(sum, zero); + row_store_64xh(&row, 32, dst, stride); +} + +void aom_dc_left_predictor_64x16_avx2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + __m128i sum = dc_sum_16_sse2(left); + (void)above; + + const __m128i eight = _mm_set1_epi16(8); + sum = _mm_add_epi16(sum, eight); + sum = _mm_srai_epi16(sum, 4); + const __m128i zero = _mm_setzero_si128(); + const __m128i r = _mm_shuffle_epi8(sum, zero); + const __m256i row = _mm256_inserti128_si256(_mm256_castsi128_si256(r), r, 1); + row_store_64xh(&row, 16, dst, stride); +} + void aom_dc_128_predictor_32x16_avx2(uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left) {
diff --git a/aom_dsp/x86/intrapred_sse2.c b/aom_dsp/x86/intrapred_sse2.c index f962c8e..56efc0e 100644 --- a/aom_dsp/x86/intrapred_sse2.c +++ b/aom_dsp/x86/intrapred_sse2.c
@@ -53,8 +53,7 @@ static INLINE void dc_store_64xh(const __m128i *row, int height, uint8_t *dst, ptrdiff_t stride) { - int i; - for (i = 0; i < height; ++i) { + for (int i = 0; i < height; ++i) { _mm_store_si128((__m128i *)dst, *row); _mm_store_si128((__m128i *)(dst + 16), *row); _mm_store_si128((__m128i *)(dst + 32), *row); @@ -95,6 +94,23 @@ return _mm_add_epi16(x0, high); } +static INLINE __m128i dc_sum_64(const uint8_t *ref) { + __m128i x0 = _mm_load_si128((__m128i const *)ref); + __m128i x1 = _mm_load_si128((__m128i const *)(ref + 16)); + __m128i x2 = _mm_load_si128((__m128i const *)(ref + 32)); + __m128i x3 = _mm_load_si128((__m128i const *)(ref + 48)); + const __m128i zero = _mm_setzero_si128(); + x0 = _mm_sad_epu8(x0, zero); + x1 = _mm_sad_epu8(x1, zero); + x2 = _mm_sad_epu8(x2, zero); + x3 = _mm_sad_epu8(x3, zero); + x0 = _mm_add_epi16(x0, x1); + x2 = _mm_add_epi16(x2, x3); + x0 = _mm_add_epi16(x0, x2); + const __m128i high = _mm_unpackhi_epi64(x0, x0); + return _mm_add_epi16(x0, high); +} + // ----------------------------------------------------------------------------- // DC_PRED @@ -345,6 +361,62 @@ dc_store_32xh(&row, 16, dst, stride); } +void aom_dc_left_predictor_32x64_sse2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + __m128i sum_left = dc_sum_64(left); + const __m128i thirtytwo = _mm_set1_epi16((uint16_t)32); + sum_left = _mm_add_epi16(sum_left, thirtytwo); + sum_left = _mm_srai_epi16(sum_left, 6); + sum_left = _mm_unpacklo_epi8(sum_left, sum_left); + sum_left = _mm_shufflelo_epi16(sum_left, 0); + const __m128i row = _mm_unpacklo_epi64(sum_left, sum_left); + dc_store_32xh(&row, 64, dst, stride); +} + +void aom_dc_left_predictor_64x64_sse2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + __m128i sum_left = dc_sum_64(left); + const __m128i thirtytwo = _mm_set1_epi16((uint16_t)32); + sum_left = _mm_add_epi16(sum_left, thirtytwo); + sum_left = _mm_srai_epi16(sum_left, 6); + sum_left = _mm_unpacklo_epi8(sum_left, sum_left); + sum_left = _mm_shufflelo_epi16(sum_left, 0); + const __m128i row = _mm_unpacklo_epi64(sum_left, sum_left); + dc_store_64xh(&row, 64, dst, stride); +} + +void aom_dc_left_predictor_64x32_sse2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + __m128i sum_left = dc_sum_32(left); + const __m128i sixteen = _mm_set1_epi16((uint16_t)16); + sum_left = _mm_add_epi16(sum_left, sixteen); + sum_left = _mm_srai_epi16(sum_left, 5); + sum_left = _mm_unpacklo_epi8(sum_left, sum_left); + sum_left = _mm_shufflelo_epi16(sum_left, 0); + const __m128i row = _mm_unpacklo_epi64(sum_left, sum_left); + dc_store_64xh(&row, 32, dst, stride); +} + +void aom_dc_left_predictor_64x16_sse2(uint8_t *dst, ptrdiff_t stride, + const uint8_t *above, + const uint8_t *left) { + (void)above; + __m128i sum_left = dc_sum_16(left); + const __m128i eight = _mm_set1_epi16((uint16_t)8); + sum_left = _mm_add_epi16(sum_left, eight); + sum_left = _mm_srai_epi16(sum_left, 4); + sum_left = _mm_unpacklo_epi8(sum_left, sum_left); + sum_left = _mm_shufflelo_epi16(sum_left, 0); + const __m128i row = _mm_unpacklo_epi64(sum_left, sum_left); + dc_store_64xh(&row, 16, dst, stride); +} + // ----------------------------------------------------------------------------- // DC_128
diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc index cd41ea2..cb1735e 100644 --- a/test/test_intra_pred_speed.cc +++ b/test/test_intra_pred_speed.cc
@@ -687,7 +687,7 @@ aom_dc_top_predictor_32x16_sse2, aom_dc_128_predictor_32x16_sse2, aom_v_predictor_32x16_sse2, aom_h_predictor_32x16_sse2, NULL, NULL, NULL, NULL) -INTRA_PRED_TEST(SSE2_3, TX_32X64, NULL, NULL, NULL, +INTRA_PRED_TEST(SSE2_3, TX_32X64, NULL, aom_dc_left_predictor_32x64_sse2, NULL, aom_dc_128_predictor_32x64_sse2, NULL, NULL, NULL, NULL, NULL, NULL) #endif // HAVE_SSE2 @@ -715,7 +715,7 @@ aom_dc_top_predictor_32x16_avx2, aom_dc_128_predictor_32x16_avx2, aom_v_predictor_32x16_avx2, NULL, aom_paeth_predictor_32x16_avx2, NULL, NULL, NULL) -INTRA_PRED_TEST(AVX2_3, TX_32X64, NULL, NULL, NULL, +INTRA_PRED_TEST(AVX2_3, TX_32X64, NULL, aom_dc_left_predictor_32x64_avx2, NULL, aom_dc_128_predictor_32x64_avx2, NULL, NULL, aom_paeth_predictor_32x64_avx2, NULL, NULL, NULL) #endif // HAVE_AVX2 @@ -760,13 +760,13 @@ aom_smooth_h_predictor_64x16_c) #if HAVE_SSE2 -INTRA_PRED_TEST(SSE2_4, TX_64X64, NULL, NULL, NULL, +INTRA_PRED_TEST(SSE2_4, TX_64X64, NULL, aom_dc_left_predictor_64x64_sse2, NULL, aom_dc_128_predictor_64x64_sse2, NULL, NULL, NULL, NULL, NULL, NULL) -INTRA_PRED_TEST(SSE2_5, TX_64X32, NULL, NULL, NULL, +INTRA_PRED_TEST(SSE2_5, TX_64X32, NULL, aom_dc_left_predictor_64x32_sse2, NULL, aom_dc_128_predictor_64x32_sse2, NULL, NULL, NULL, NULL, NULL, NULL) -INTRA_PRED_TEST(SSE2_6, TX_64X16, NULL, NULL, NULL, +INTRA_PRED_TEST(SSE2_6, TX_64X16, NULL, aom_dc_left_predictor_64x16_sse2, NULL, aom_dc_128_predictor_64x16_sse2, NULL, NULL, NULL, NULL, NULL, NULL) #endif @@ -781,13 +781,13 @@ #endif #if HAVE_AVX2 -INTRA_PRED_TEST(AVX2_4, TX_64X64, NULL, NULL, NULL, +INTRA_PRED_TEST(AVX2_4, TX_64X64, NULL, aom_dc_left_predictor_64x64_avx2, NULL, aom_dc_128_predictor_64x64_avx2, NULL, NULL, aom_paeth_predictor_64x64_avx2, NULL, NULL, NULL) -INTRA_PRED_TEST(AVX2_5, TX_64X32, NULL, NULL, NULL, +INTRA_PRED_TEST(AVX2_5, TX_64X32, NULL, aom_dc_left_predictor_64x32_avx2, NULL, aom_dc_128_predictor_64x32_avx2, NULL, NULL, aom_paeth_predictor_64x32_avx2, NULL, NULL, NULL) -INTRA_PRED_TEST(AVX2_6, TX_64X16, NULL, NULL, NULL, +INTRA_PRED_TEST(AVX2_6, TX_64X16, NULL, aom_dc_left_predictor_64x16_avx2, NULL, aom_dc_128_predictor_64x16_avx2, NULL, NULL, aom_paeth_predictor_64x16_avx2, NULL, NULL, NULL) #endif