| /* |
| * Copyright (c) 2018, Alliance for Open Media. All rights reserved |
| * |
| * This source code is subject to the terms of the BSD 2 Clause License and |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| * was not distributed with this source code in the LICENSE file, you can |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| * Media Patent License 1.0 was not distributed with this source code in the |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| */ |
| |
| #include <immintrin.h> // AVX2 |
| #include "aom_dsp/x86/synonyms.h" |
| #include "aom_dsp/x86/synonyms_avx2.h" |
| #include "aom_dsp/x86/transpose_sse2.h" |
| |
| #include "config/av1_rtcd.h" |
| #include "av1/common/restoration.h" |
| #include "av1/encoder/pickrst.h" |
| |
| static INLINE void acc_stat_avx2(int32_t *dst, const uint8_t *src, |
| const __m128i *shuffle, const __m256i *kl) { |
| const __m128i s = _mm_shuffle_epi8(xx_loadu_128(src), *shuffle); |
| const __m256i d0 = _mm256_madd_epi16(*kl, _mm256_cvtepu8_epi16(s)); |
| const __m256i dst0 = yy_loadu_256(dst); |
| const __m256i r0 = _mm256_add_epi32(dst0, d0); |
| yy_storeu_256(dst, r0); |
| } |
| |
| static INLINE void acc_stat_win7_one_line_avx2( |
| const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, |
| int dgd_stride, const __m128i *shuffle, int32_t *sumX, |
| int32_t sumY[WIENER_WIN][WIENER_WIN], int32_t M_int[WIENER_WIN][WIENER_WIN], |
| int32_t H_int[WIENER_WIN2][WIENER_WIN * 8]) { |
| int j, k, l; |
| const int wiener_win = WIENER_WIN; |
| for (j = h_start; j < h_end; j += 2) { |
| const uint8_t X1 = src[j]; |
| const uint8_t X2 = src[j + 1]; |
| *sumX += X1 + X2; |
| const uint8_t *dgd_ij = dgd + j; |
| for (k = 0; k < wiener_win; k++) { |
| const uint8_t *dgd_ijk = dgd_ij + k * dgd_stride; |
| for (l = 0; l < wiener_win; l++) { |
| int32_t *H_ = &H_int[(l * wiener_win + k)][0]; |
| const uint8_t D1 = dgd_ijk[l]; |
| const uint8_t D2 = dgd_ijk[l + 1]; |
| sumY[k][l] += D1 + D2; |
| M_int[k][l] += D1 * X1 + D2 * X2; |
| |
| const __m256i kl = |
| _mm256_cvtepu8_epi16(_mm_set1_epi16(*((uint16_t *)(dgd_ijk + l)))); |
| acc_stat_avx2(H_ + 0 * 8, dgd_ij + 0 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 1 * 8, dgd_ij + 1 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 2 * 8, dgd_ij + 2 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 3 * 8, dgd_ij + 3 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 4 * 8, dgd_ij + 4 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 5 * 8, dgd_ij + 5 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 6 * 8, dgd_ij + 6 * dgd_stride, shuffle, &kl); |
| } |
| } |
| } |
| } |
| |
| static INLINE void compute_stats_win7_opt_avx2( |
| const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, |
| int v_end, int dgd_stride, int src_stride, double *M, double *H) { |
| int i, j, k, l, m, n; |
| const int wiener_win = WIENER_WIN; |
| const int pixel_count = (h_end - h_start) * (v_end - v_start); |
| const int wiener_win2 = wiener_win * wiener_win; |
| const int wiener_halfwin = (wiener_win >> 1); |
| const double avg = |
| find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); |
| |
| int32_t M_int32[WIENER_WIN][WIENER_WIN] = { { 0 } }; |
| int64_t M_int64[WIENER_WIN][WIENER_WIN] = { { 0 } }; |
| int32_t H_int32[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; |
| int64_t H_int64[WIENER_WIN2][WIENER_WIN * 8] = { { 0 } }; |
| int32_t sumY[WIENER_WIN][WIENER_WIN] = { { 0 } }; |
| int32_t sumX = 0; |
| const uint8_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; |
| |
| const __m128i shuffle = xx_loadu_128(g_shuffle_stats_data); |
| for (j = v_start; j < v_end; j += 64) { |
| const int vert_end = AOMMIN(64, v_end - j) + j; |
| for (i = j; i < vert_end; i++) { |
| acc_stat_win7_one_line_avx2( |
| dgd_win + i * dgd_stride, src + i * src_stride, h_start, h_end, |
| dgd_stride, &shuffle, &sumX, sumY, M_int32, H_int32); |
| } |
| for (k = 0; k < wiener_win; ++k) { |
| for (l = 0; l < wiener_win; ++l) { |
| M_int64[k][l] += M_int32[k][l]; |
| M_int32[k][l] = 0; |
| } |
| } |
| for (k = 0; k < WIENER_WIN2; ++k) { |
| for (l = 0; l < WIENER_WIN * 8; ++l) { |
| H_int64[k][l] += H_int32[k][l]; |
| H_int32[k][l] = 0; |
| } |
| } |
| } |
| |
| const double avg_square_sum = avg * avg * pixel_count; |
| for (k = 0; k < wiener_win; k++) { |
| for (l = 0; l < wiener_win; l++) { |
| const int32_t idx0 = l * wiener_win + k; |
| M[idx0] = M_int64[k][l] + avg_square_sum - avg * (sumX + sumY[k][l]); |
| double *H_ = H + idx0 * wiener_win2; |
| int64_t *H_int_ = &H_int64[idx0][0]; |
| for (m = 0; m < wiener_win; m++) { |
| for (n = 0; n < wiener_win; n++) { |
| H_[m * wiener_win + n] = H_int_[n * 8 + m] + avg_square_sum - |
| avg * (sumY[k][l] + sumY[n][m]); |
| } |
| } |
| } |
| } |
| } |
| |
| static INLINE void acc_stat_win5_one_line_avx2( |
| const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, |
| int dgd_stride, const __m128i *shuffle, int32_t *sumX, |
| int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA], |
| int32_t M_int[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA], |
| int32_t H_int[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8]) { |
| int j, k, l; |
| const int wiener_win = WIENER_WIN_CHROMA; |
| for (j = h_start; j < h_end; j += 2) { |
| const uint8_t X1 = src[j]; |
| const uint8_t X2 = src[j + 1]; |
| *sumX += X1 + X2; |
| const uint8_t *dgd_ij = dgd + j; |
| for (k = 0; k < wiener_win; k++) { |
| const uint8_t *dgd_ijk = dgd_ij + k * dgd_stride; |
| for (l = 0; l < wiener_win; l++) { |
| int32_t *H_ = &H_int[(l * wiener_win + k)][0]; |
| const uint8_t D1 = dgd_ijk[l]; |
| const uint8_t D2 = dgd_ijk[l + 1]; |
| sumY[k][l] += D1 + D2; |
| M_int[k][l] += D1 * X1 + D2 * X2; |
| |
| const __m256i kl = |
| _mm256_cvtepu8_epi16(_mm_set1_epi16(*((uint16_t *)(dgd_ijk + l)))); |
| acc_stat_avx2(H_ + 0 * 8, dgd_ij + 0 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 1 * 8, dgd_ij + 1 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 2 * 8, dgd_ij + 2 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 3 * 8, dgd_ij + 3 * dgd_stride, shuffle, &kl); |
| acc_stat_avx2(H_ + 4 * 8, dgd_ij + 4 * dgd_stride, shuffle, &kl); |
| } |
| } |
| } |
| } |
| |
| static INLINE void compute_stats_win5_opt_avx2( |
| const uint8_t *dgd, const uint8_t *src, int h_start, int h_end, int v_start, |
| int v_end, int dgd_stride, int src_stride, double *M, double *H) { |
| int i, j, k, l, m, n; |
| const int wiener_win = WIENER_WIN_CHROMA; |
| const int pixel_count = (h_end - h_start) * (v_end - v_start); |
| const int wiener_win2 = wiener_win * wiener_win; |
| const int wiener_halfwin = (wiener_win >> 1); |
| const double avg = |
| find_average(dgd, h_start, h_end, v_start, v_end, dgd_stride); |
| |
| int32_t M_int32[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; |
| int64_t M_int64[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; |
| int32_t H_int32[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; |
| int64_t H_int64[WIENER_WIN2_CHROMA][WIENER_WIN_CHROMA * 8] = { { 0 } }; |
| int32_t sumY[WIENER_WIN_CHROMA][WIENER_WIN_CHROMA] = { { 0 } }; |
| int32_t sumX = 0; |
| const uint8_t *dgd_win = dgd - wiener_halfwin * dgd_stride - wiener_halfwin; |
| |
| const __m128i shuffle = xx_loadu_128(g_shuffle_stats_data); |
| for (j = v_start; j < v_end; j += 64) { |
| const int vert_end = AOMMIN(64, v_end - j) + j; |
| for (i = j; i < vert_end; i++) { |
| acc_stat_win5_one_line_avx2( |
| dgd_win + i * dgd_stride, src + i * src_stride, h_start, h_end, |
| dgd_stride, &shuffle, &sumX, sumY, M_int32, H_int32); |
| } |
| for (k = 0; k < wiener_win; ++k) { |
| for (l = 0; l < wiener_win; ++l) { |
| M_int64[k][l] += M_int32[k][l]; |
| M_int32[k][l] = 0; |
| } |
| } |
| for (k = 0; k < WIENER_WIN2_CHROMA; ++k) { |
| for (l = 0; l < WIENER_WIN_CHROMA * 8; ++l) { |
| H_int64[k][l] += H_int32[k][l]; |
| H_int32[k][l] = 0; |
| } |
| } |
| } |
| |
| const double avg_square_sum = avg * avg * pixel_count; |
| for (k = 0; k < wiener_win; k++) { |
| for (l = 0; l < wiener_win; l++) { |
| const int32_t idx0 = l * wiener_win + k; |
| M[idx0] = M_int64[k][l] + avg_square_sum - avg * (sumX + sumY[k][l]); |
| double *H_ = H + idx0 * wiener_win2; |
| int64_t *H_int_ = &H_int64[idx0][0]; |
| for (m = 0; m < wiener_win; m++) { |
| for (n = 0; n < wiener_win; n++) { |
| H_[m * wiener_win + n] = H_int_[n * 8 + m] + avg_square_sum - |
| avg * (sumY[k][l] + sumY[n][m]); |
| } |
| } |
| } |
| } |
| } |
| |
| void compute_stats_avx2(int wiener_win, const uint8_t *dgd, const uint8_t *src, |
| int h_start, int h_end, int v_start, int v_end, |
| int dgd_stride, int src_stride, double *M, double *H) { |
| if (wiener_win == WIENER_WIN) { |
| compute_stats_win7_opt_avx2(dgd, src, h_start, h_end, v_start, v_end, |
| dgd_stride, src_stride, M, H); |
| } else if (wiener_win == WIENER_WIN_CHROMA) { |
| compute_stats_win5_opt_avx2(dgd, src, h_start, h_end, v_start, v_end, |
| dgd_stride, src_stride, M, H); |
| } else { |
| compute_stats_c(wiener_win, dgd, src, h_start, h_end, v_start, v_end, |
| dgd_stride, src_stride, M, H); |
| } |
| } |
