blob: b230cd9ee711d53a1f708601831f7138ef4345fb [file] [log] [blame] [edit]
/*
* Copyright (c) 2021, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 3-Clause Clear License
* and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
* License was not distributed with this source code in the LICENSE file, you
* can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the
* Alliance for Open Media Patent License 1.0 was not distributed with this
* source code in the PATENTS file, you can obtain it at
* aomedia.org/license/patent-license/.
*/
#include <assert.h>
#include <smmintrin.h>
#include "config/aom_config.h"
#include "aom_ports/mem.h"
#include "aom/aom_integer.h"
#include "aom_dsp/x86/synonyms.h"
static INLINE int64_t summary_all_sse4(const __m128i *sum_all) {
int64_t sum;
const __m128i sum0 = _mm_cvtepu32_epi64(*sum_all);
const __m128i sum1 = _mm_cvtepu32_epi64(_mm_srli_si128(*sum_all, 8));
const __m128i sum_2x64 = _mm_add_epi64(sum0, sum1);
const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8));
xx_storel_64(&sum, sum_1x64);
return sum;
}
static INLINE void summary_32_sse4(const __m128i *sum32, __m128i *sum64) {
const __m128i sum0 = _mm_cvtepu32_epi64(*sum32);
const __m128i sum1 = _mm_cvtepu32_epi64(_mm_srli_si128(*sum32, 8));
*sum64 = _mm_add_epi64(sum0, *sum64);
*sum64 = _mm_add_epi64(sum1, *sum64);
}
static INLINE void highbd_sse_w4x2_sse4_1(__m128i *sum, const uint16_t *a,
int a_stride, const uint16_t *b,
int b_stride) {
const __m128i v_a0 = xx_loadl_64(a);
const __m128i v_a1 = xx_loadl_64(a + a_stride);
const __m128i v_b0 = xx_loadl_64(b);
const __m128i v_b1 = xx_loadl_64(b + b_stride);
const __m128i v_a_w = _mm_unpacklo_epi64(v_a0, v_a1);
const __m128i v_b_w = _mm_unpacklo_epi64(v_b0, v_b1);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
static INLINE void highbd_sse_w8_sse4_1(__m128i *sum, const uint16_t *a,
const uint16_t *b) {
const __m128i v_a_w = xx_loadu_128(a);
const __m128i v_b_w = xx_loadu_128(b);
const __m128i v_d_w = _mm_sub_epi16(v_a_w, v_b_w);
*sum = _mm_add_epi32(*sum, _mm_madd_epi16(v_d_w, v_d_w));
}
int64_t aom_highbd_sse_sse4_1(const uint16_t *a, int a_stride,
const uint16_t *b, int b_stride, int width,
int height) {
int32_t y = 0;
int64_t sse = 0;
__m128i sum = _mm_setzero_si128();
switch (width) {
case 4:
do {
highbd_sse_w4x2_sse4_1(&sum, a, a_stride, b, b_stride);
a += a_stride << 1;
b += b_stride << 1;
y += 2;
} while (y < height);
sse = summary_all_sse4(&sum);
break;
case 8:
do {
highbd_sse_w8_sse4_1(&sum, a, b);
a += a_stride;
b += b_stride;
y += 1;
} while (y < height);
sse = summary_all_sse4(&sum);
break;
case 16:
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8, b + 8);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 64 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 64;
} while (y < height);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 32:
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 32 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 32;
} while (y < height);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 64:
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 16 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 16;
} while (y < height);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
case 128:
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
highbd_sse_w8_sse4_1(&sum32, a, b);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 1, b + 8 * 1);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 2, b + 8 * 2);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 3, b + 8 * 3);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 4, b + 8 * 4);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 5, b + 8 * 5);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 6, b + 8 * 6);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 7, b + 8 * 7);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 8, b + 8 * 8);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 9, b + 8 * 9);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 10, b + 8 * 10);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 11, b + 8 * 11);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 12, b + 8 * 12);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 13, b + 8 * 13);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 14, b + 8 * 14);
highbd_sse_w8_sse4_1(&sum32, a + 8 * 15, b + 8 * 15);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
default:
if (width & 0x7) {
do {
__m128i sum32 = _mm_setzero_si128();
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
highbd_sse_w8_sse4_1(&sum32, a + i + a_stride, b + i + b_stride);
i += 8;
} while (i + 4 < width);
highbd_sse_w4x2_sse4_1(&sum32, a + i, a_stride, b + i, b_stride);
a += (a_stride << 1);
b += (b_stride << 1);
y += 2;
summary_32_sse4(&sum32, &sum);
} while (y < height);
} else {
do {
int l = 0;
__m128i sum32 = _mm_setzero_si128();
do {
int i = 0;
do {
highbd_sse_w8_sse4_1(&sum32, a + i, b + i);
i += 8;
} while (i < width);
a += a_stride;
b += b_stride;
l += 1;
} while (l < 8 && l < (height - y));
summary_32_sse4(&sum32, &sum);
y += 8;
} while (y < height);
}
xx_storel_64(&sse, _mm_add_epi64(sum, _mm_srli_si128(sum, 8)));
break;
}
return sse;
}