| /* |
| * Copyright (c) 2021, Alliance for Open Media. All rights reserved |
| * |
| * This source code is subject to the terms of the BSD 3-Clause Clear License |
| * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear |
| * License was not distributed with this source code in the LICENSE file, you |
| * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the |
| * Alliance for Open Media Patent License 1.0 was not distributed with this |
| * source code in the PATENTS file, you can obtain it at |
| * aomedia.org/license/patent-license/. |
| */ |
| |
| #include <assert.h> |
| #include <emmintrin.h> // SSE2 |
| #include <tmmintrin.h> |
| |
| #include "config/aom_config.h" |
| #include "config/aom_dsp_rtcd.h" |
| #include "config/av1_rtcd.h" |
| |
| #include "aom_dsp/x86/synonyms.h" |
| |
| unsigned int aom_sad4xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, |
| int b_stride, int width, int height) { |
| int i; |
| assert(width == 4); |
| (void)width; |
| |
| __m128i sad = _mm_setzero_si128(); |
| for (i = 0; i < height; i += 4) { |
| __m128i x0 = xx_loadl_32(a + 0 * a_stride); |
| __m128i x1 = xx_loadl_32(a + 1 * a_stride); |
| __m128i x2 = xx_loadl_32(a + 2 * a_stride); |
| __m128i x3 = xx_loadl_32(a + 3 * a_stride); |
| __m128i x_lo = _mm_unpacklo_epi32(x0, x1); |
| __m128i x_hi = _mm_unpacklo_epi32(x2, x3); |
| |
| __m128i x = _mm_unpacklo_epi64(x_lo, x_hi); |
| |
| x0 = xx_loadl_32(b + 0 * b_stride); |
| x1 = xx_loadl_32(b + 1 * b_stride); |
| x2 = xx_loadl_32(b + 2 * b_stride); |
| x3 = xx_loadl_32(b + 3 * b_stride); |
| x_lo = _mm_unpacklo_epi32(x0, x1); |
| x_hi = _mm_unpacklo_epi32(x2, x3); |
| |
| __m128i y = _mm_unpacklo_epi64(x_lo, x_hi); |
| |
| __m128i sad4x4 = _mm_sad_epu8(x, y); |
| sad = _mm_add_epi32(sad, sad4x4); |
| |
| a += 4 * a_stride; |
| b += 4 * b_stride; |
| } |
| |
| // At this point, we have two 32-bit partial SADs at bit[0:31] and [64:95]. |
| const unsigned int res = |
| _mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8)); |
| |
| return res; |
| } |
| |
| unsigned int aom_sad8xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, |
| int b_stride, int width, int height) { |
| int i; |
| assert(width == 8); |
| (void)width; |
| |
| __m128i sad = _mm_setzero_si128(); |
| for (i = 0; i < height; i += 2) { |
| __m128i x0 = xx_loadl_64(a + 0 * a_stride); |
| __m128i x1 = xx_loadl_64(a + 1 * a_stride); |
| |
| __m128i x = _mm_unpacklo_epi64(x0, x1); |
| |
| x0 = xx_loadl_64(b + 0 * b_stride); |
| x1 = xx_loadl_64(b + 1 * b_stride); |
| |
| __m128i y = _mm_unpacklo_epi64(x0, x1); |
| |
| __m128i sad8x2 = _mm_sad_epu8(x, y); |
| sad = _mm_add_epi32(sad, sad8x2); |
| |
| a += 2 * a_stride; |
| b += 2 * b_stride; |
| } |
| |
| const unsigned int res = |
| _mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8)); |
| |
| return res; |
| } |
| |
| unsigned int aom_sad16xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, |
| int b_stride, int width, int height) { |
| int i; |
| assert(width == 16); |
| (void)width; |
| |
| __m128i sad = _mm_setzero_si128(); |
| for (i = 0; i < height; ++i) { |
| __m128i x = xx_loadu_128(a); |
| __m128i y = xx_loadu_128(b); |
| |
| __m128i sad16x1 = _mm_sad_epu8(x, y); |
| sad = _mm_add_epi32(sad, sad16x1); |
| |
| a += a_stride; |
| b += b_stride; |
| } |
| |
| const unsigned int res = |
| _mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8)); |
| |
| return res; |
| } |
| |
| unsigned int aom_sad32xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, |
| int b_stride, int width, int height) { |
| int i, j; |
| assert(width == 32); |
| (void)width; |
| |
| __m128i sad = _mm_setzero_si128(); |
| for (i = 0; i < height; ++i) { |
| for (j = 0; j < 2; ++j) { |
| __m128i x = xx_loadu_128(a + j * 16); |
| __m128i y = xx_loadu_128(b + j * 16); |
| |
| __m128i sad32_half = _mm_sad_epu8(x, y); |
| sad = _mm_add_epi32(sad, sad32_half); |
| } |
| |
| a += a_stride; |
| b += b_stride; |
| } |
| |
| const unsigned int res = |
| _mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8)); |
| |
| return res; |
| } |
| |
| unsigned int aom_sad64xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, |
| int b_stride, int width, int height) { |
| int i, j; |
| assert(width == 64); |
| (void)width; |
| |
| __m128i sad = _mm_setzero_si128(); |
| for (i = 0; i < height; ++i) { |
| for (j = 0; j < 4; ++j) { |
| __m128i x = xx_loadu_128(a + j * 16); |
| __m128i y = xx_loadu_128(b + j * 16); |
| |
| __m128i sad64_quarter = _mm_sad_epu8(x, y); |
| sad = _mm_add_epi32(sad, sad64_quarter); |
| } |
| |
| a += a_stride; |
| b += b_stride; |
| } |
| |
| const unsigned int res = |
| _mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8)); |
| |
| return res; |
| } |
| |
| unsigned int aom_sad128xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b, |
| int b_stride, int width, int height) { |
| int i, j; |
| assert(width == 128); |
| (void)width; |
| |
| __m128i sad = _mm_setzero_si128(); |
| for (i = 0; i < height; ++i) { |
| for (j = 0; j < 8; ++j) { |
| __m128i x = xx_loadu_128(a + j * 16); |
| __m128i y = xx_loadu_128(b + j * 16); |
| |
| __m128i sad64_quarter = _mm_sad_epu8(x, y); |
| sad = _mm_add_epi32(sad, sad64_quarter); |
| } |
| |
| a += a_stride; |
| b += b_stride; |
| } |
| |
| const unsigned int res = |
| _mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8)); |
| |
| return res; |
| } |
| |
| #define dist_wtd_sadMxN_sse2(m, n) \ |
| unsigned int aom_dist_wtd_sad##m##x##n##_avg_ssse3( \ |
| const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ |
| const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ |
| uint8_t comp_pred[m * n]; \ |
| aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \ |
| jcp_param); \ |
| return aom_sad##m##xh_sse2(src, src_stride, comp_pred, m, m, n); \ |
| } |
| |
| #define dist_wtd_sadMxN_avx2(m, n) \ |
| unsigned int aom_dist_wtd_sad##m##x##n##_avg_avx2( \ |
| const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, \ |
| const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param) { \ |
| uint8_t comp_pred[m * n]; \ |
| aom_dist_wtd_comp_avg_pred(comp_pred, second_pred, m, n, ref, ref_stride, \ |
| jcp_param); \ |
| return aom_sad##m##xh_avx2(src, src_stride, comp_pred, m, m, n); \ |
| } |
| |
| /* clang-format off */ |
| dist_wtd_sadMxN_sse2(128, 128) |
| dist_wtd_sadMxN_sse2(128, 64) |
| dist_wtd_sadMxN_sse2(64, 128) |
| dist_wtd_sadMxN_sse2(64, 64) |
| dist_wtd_sadMxN_sse2(64, 32) |
| dist_wtd_sadMxN_sse2(32, 64) |
| dist_wtd_sadMxN_sse2(32, 32) |
| dist_wtd_sadMxN_sse2(32, 16) |
| dist_wtd_sadMxN_sse2(16, 32) |
| dist_wtd_sadMxN_sse2(16, 16) |
| dist_wtd_sadMxN_sse2(16, 8) |
| dist_wtd_sadMxN_sse2(8, 16) |
| dist_wtd_sadMxN_sse2(8, 8) |
| dist_wtd_sadMxN_sse2(8, 4) |
| dist_wtd_sadMxN_sse2(4, 8) |
| dist_wtd_sadMxN_sse2(4, 4) |
| dist_wtd_sadMxN_sse2(4, 16) |
| dist_wtd_sadMxN_sse2(16, 4) |
| dist_wtd_sadMxN_sse2(8, 32) |
| dist_wtd_sadMxN_sse2(32, 8) |
| dist_wtd_sadMxN_sse2(16, 64) |
| dist_wtd_sadMxN_sse2(64, 16) |
| /* clang-format on */ |