blob: 7ef5eb5b192a8cc6612f2cea87d77d3724ec469f [file] [log] [blame]
/*
* Copyright (c) 2021, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 3-Clause Clear License
* and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
* License was not distributed with this source code in the LICENSE file, you
* can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the
* Alliance for Open Media Patent License 1.0 was not distributed with this
* source code in the PATENTS file, you can obtain it at
* aomedia.org/license/patent-license/.
*/
#include <assert.h>
#include <emmintrin.h> // SSE2
#include <tmmintrin.h>
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
#include "config/av1_rtcd.h"
#include "aom_dsp/x86/synonyms.h"
unsigned int aom_sad4xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int i;
assert(width == 4);
(void)width;
__m128i sad = _mm_setzero_si128();
for (i = 0; i < height; i += 4) {
__m128i x0 = xx_loadl_32(a + 0 * a_stride);
__m128i x1 = xx_loadl_32(a + 1 * a_stride);
__m128i x2 = xx_loadl_32(a + 2 * a_stride);
__m128i x3 = xx_loadl_32(a + 3 * a_stride);
__m128i x_lo = _mm_unpacklo_epi32(x0, x1);
__m128i x_hi = _mm_unpacklo_epi32(x2, x3);
__m128i x = _mm_unpacklo_epi64(x_lo, x_hi);
x0 = xx_loadl_32(b + 0 * b_stride);
x1 = xx_loadl_32(b + 1 * b_stride);
x2 = xx_loadl_32(b + 2 * b_stride);
x3 = xx_loadl_32(b + 3 * b_stride);
x_lo = _mm_unpacklo_epi32(x0, x1);
x_hi = _mm_unpacklo_epi32(x2, x3);
__m128i y = _mm_unpacklo_epi64(x_lo, x_hi);
__m128i sad4x4 = _mm_sad_epu8(x, y);
sad = _mm_add_epi32(sad, sad4x4);
a += 4 * a_stride;
b += 4 * b_stride;
}
// At this point, we have two 32-bit partial SADs at bit[0:31] and [64:95].
const unsigned int res =
_mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8));
return res;
}
unsigned int aom_sad8xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int i;
assert(width == 8);
(void)width;
__m128i sad = _mm_setzero_si128();
for (i = 0; i < height; i += 2) {
__m128i x0 = xx_loadl_64(a + 0 * a_stride);
__m128i x1 = xx_loadl_64(a + 1 * a_stride);
__m128i x = _mm_unpacklo_epi64(x0, x1);
x0 = xx_loadl_64(b + 0 * b_stride);
x1 = xx_loadl_64(b + 1 * b_stride);
__m128i y = _mm_unpacklo_epi64(x0, x1);
__m128i sad8x2 = _mm_sad_epu8(x, y);
sad = _mm_add_epi32(sad, sad8x2);
a += 2 * a_stride;
b += 2 * b_stride;
}
const unsigned int res =
_mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8));
return res;
}
unsigned int aom_sad16xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int i;
assert(width == 16);
(void)width;
__m128i sad = _mm_setzero_si128();
for (i = 0; i < height; ++i) {
__m128i x = xx_loadu_128(a);
__m128i y = xx_loadu_128(b);
__m128i sad16x1 = _mm_sad_epu8(x, y);
sad = _mm_add_epi32(sad, sad16x1);
a += a_stride;
b += b_stride;
}
const unsigned int res =
_mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8));
return res;
}
unsigned int aom_sad32xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int i, j;
assert(width == 32);
(void)width;
__m128i sad = _mm_setzero_si128();
for (i = 0; i < height; ++i) {
for (j = 0; j < 2; ++j) {
__m128i x = xx_loadu_128(a + j * 16);
__m128i y = xx_loadu_128(b + j * 16);
__m128i sad32_half = _mm_sad_epu8(x, y);
sad = _mm_add_epi32(sad, sad32_half);
}
a += a_stride;
b += b_stride;
}
const unsigned int res =
_mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8));
return res;
}
unsigned int aom_sad64xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int i, j;
assert(width == 64);
(void)width;
__m128i sad = _mm_setzero_si128();
for (i = 0; i < height; ++i) {
for (j = 0; j < 4; ++j) {
__m128i x = xx_loadu_128(a + j * 16);
__m128i y = xx_loadu_128(b + j * 16);
__m128i sad64_quarter = _mm_sad_epu8(x, y);
sad = _mm_add_epi32(sad, sad64_quarter);
}
a += a_stride;
b += b_stride;
}
const unsigned int res =
_mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8));
return res;
}
unsigned int aom_sad128xh_sse2(const uint8_t *a, int a_stride, const uint8_t *b,
int b_stride, int width, int height) {
int i, j;
assert(width == 128);
(void)width;
__m128i sad = _mm_setzero_si128();
for (i = 0; i < height; ++i) {
for (j = 0; j < 8; ++j) {
__m128i x = xx_loadu_128(a + j * 16);
__m128i y = xx_loadu_128(b + j * 16);
__m128i sad64_quarter = _mm_sad_epu8(x, y);
sad = _mm_add_epi32(sad, sad64_quarter);
}
a += a_stride;
b += b_stride;
}
const unsigned int res =
_mm_cvtsi128_si32(sad) + _mm_cvtsi128_si32(_mm_srli_si128(sad, 8));
return res;
}
/* clang-format on */