remove third_party/SVT-AV1 The native convolve optimizations are neutral to ~1% faster when testing 2 pass speed 2 VBR, and realtime CBR with speeds 5, 10, and 11. Bug: 446258249 Change-Id: Ica875d3d17ffb53c3c772616ce4c5cd556dede74
diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index 43ffe68..0738274 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake
@@ -97,15 +97,6 @@ "${AOM_ROOT}/aom_dsp/x86/blend_a64_mask_avx2.c" "${AOM_ROOT}/aom_dsp/x86/bitdepth_conversion_avx2.h") -if(CONFIG_SVT_AV1) - list(APPEND AOM_DSP_COMMON_INTRIN_AVX2 - "${AOM_ROOT}/third_party/SVT-AV1/convolve_2d_avx2.h" - "${AOM_ROOT}/third_party/SVT-AV1/convolve_avx2.h" - "${AOM_ROOT}/third_party/SVT-AV1/EbMemory_AVX2.h" - "${AOM_ROOT}/third_party/SVT-AV1/EbMemory_SSE4_1.h" - "${AOM_ROOT}/third_party/SVT-AV1/synonyms.h") -endif() - list(APPEND AOM_DSP_COMMON_INTRIN_NEON "${AOM_ROOT}/aom_dsp/arm/aom_convolve_copy_neon.c" "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon.c"
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake index 1ca6310..05113fb 100644 --- a/build/cmake/aom_config_defaults.cmake +++ b/build/cmake/aom_config_defaults.cmake
@@ -82,9 +82,6 @@ set_aom_config_var(CONFIG_GCOV 0 "Enable gcov support.") set_aom_config_var(CONFIG_GPROF 0 "Enable gprof support.") set_aom_config_var(CONFIG_LIBYUV 1 "Enables libyuv scaling/conversion support.") -# Set CONFIG_SVT_AV1 to 0 to avoid the BSD 3-Clause Clear License used by the -# code in third_party/SVT-AV1/. -set_aom_config_var(CONFIG_SVT_AV1 1 "Enables SVT-AV1 AVX2 convolution support.") set_aom_config_var(CONFIG_AV1_HIGHBITDEPTH 1 "Build with high bitdepth support.")
diff --git a/third_party/SVT-AV1/EbMemory_AVX2.h b/third_party/SVT-AV1/EbMemory_AVX2.h deleted file mode 100644 index bd0c4fb..0000000 --- a/third_party/SVT-AV1/EbMemory_AVX2.h +++ /dev/null
@@ -1,110 +0,0 @@ -/* - * Copyright(c) 2019 Intel Corporation - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at https://www.aomedia.org/license/software-license. If the - * Alliance for Open Media Patent License 1.0 was not distributed with this - * source code in the PATENTS file, you can obtain it at - * https://www.aomedia.org/license/patent-license. - */ - -#ifndef AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_AVX2_H_ -#define AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_AVX2_H_ - -#include <immintrin.h> - -#include "config/aom_config.h" - -#include "aom/aom_integer.h" - -#ifndef _mm256_set_m128i -#define _mm256_set_m128i(/* __m128i */ hi, /* __m128i */ lo) \ - _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 0x1) -#endif - -#ifndef _mm256_setr_m128i -#define _mm256_setr_m128i(/* __m128i */ lo, /* __m128i */ hi) \ - _mm256_set_m128i((hi), (lo)) -#endif - -static inline __m256i load_u8_4x2_avx2(const uint8_t *const src, - const ptrdiff_t stride) { - __m128i src01; - src01 = _mm_cvtsi32_si128(*(int32_t *)(src + 0 * stride)); - src01 = _mm_insert_epi32(src01, *(int32_t *)(src + 1 * stride), 1); - return _mm256_setr_m128i(src01, _mm_setzero_si128()); -} - -static inline __m256i load_u8_4x4_avx2(const uint8_t *const src, - const ptrdiff_t stride) { - __m128i src01, src23; - src01 = _mm_cvtsi32_si128(*(int32_t *)(src + 0 * stride)); - src01 = _mm_insert_epi32(src01, *(int32_t *)(src + 1 * stride), 1); - src23 = _mm_cvtsi32_si128(*(int32_t *)(src + 2 * stride)); - src23 = _mm_insert_epi32(src23, *(int32_t *)(src + 3 * stride), 1); - return _mm256_setr_m128i(src01, src23); -} - -static inline __m256i load_u8_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride) { - const __m128i src0 = _mm_loadl_epi64((__m128i *)(src + 0 * stride)); - const __m128i src1 = _mm_loadl_epi64((__m128i *)(src + 1 * stride)); - return _mm256_setr_m128i(src0, src1); -} - -static inline __m256i load_u8_8x4_avx2(const uint8_t *const src, - const ptrdiff_t stride) { - __m128i src01, src23; - src01 = _mm_loadl_epi64((__m128i *)(src + 0 * stride)); - src01 = _mm_castpd_si128(_mm_loadh_pd(_mm_castsi128_pd(src01), - (double *)(void *)(src + 1 * stride))); - src23 = _mm_loadl_epi64((__m128i *)(src + 2 * stride)); - src23 = _mm_castpd_si128(_mm_loadh_pd(_mm_castsi128_pd(src23), - (double *)(void *)(src + 3 * stride))); - return _mm256_setr_m128i(src01, src23); -} - -static inline __m256i loadu_8bit_16x2_avx2(const void *const src, - const ptrdiff_t strideInByte) { - const __m128i src0 = _mm_loadu_si128((__m128i *)src); - const __m128i src1 = - _mm_loadu_si128((__m128i *)((uint8_t *)src + strideInByte)); - return _mm256_setr_m128i(src0, src1); -} - -static inline __m256i loadu_u8_16x2_avx2(const uint8_t *const src, - const ptrdiff_t stride) { - return loadu_8bit_16x2_avx2(src, sizeof(*src) * stride); -} - -static inline __m256i loadu_u16_8x2_avx2(const uint16_t *const src, - const ptrdiff_t stride) { - return loadu_8bit_16x2_avx2(src, sizeof(*src) * stride); -} - -static inline void storeu_8bit_16x2_avx2(const __m256i src, void *const dst, - const ptrdiff_t strideInByte) { - const __m128i d0 = _mm256_castsi256_si128(src); - const __m128i d1 = _mm256_extracti128_si256(src, 1); - _mm_storeu_si128((__m128i *)dst, d0); - _mm_storeu_si128((__m128i *)((uint8_t *)dst + strideInByte), d1); -} - -static inline void storeu_u8_16x2_avx2(const __m256i src, uint8_t *const dst, - const ptrdiff_t stride) { - storeu_8bit_16x2_avx2(src, dst, sizeof(*dst) * stride); -} - -static inline void storeu_s16_8x2_avx2(const __m256i src, int16_t *const dst, - const ptrdiff_t stride) { - storeu_8bit_16x2_avx2(src, dst, sizeof(*dst) * stride); -} - -static inline void storeu_u16_8x2_avx2(const __m256i src, uint16_t *const dst, - const ptrdiff_t stride) { - storeu_8bit_16x2_avx2(src, dst, sizeof(*dst) * stride); -} - -#endif // AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_AVX2_H_
diff --git a/third_party/SVT-AV1/EbMemory_SSE4_1.h b/third_party/SVT-AV1/EbMemory_SSE4_1.h deleted file mode 100644 index a1ab531..0000000 --- a/third_party/SVT-AV1/EbMemory_SSE4_1.h +++ /dev/null
@@ -1,39 +0,0 @@ -/* - * Copyright(c) 2019 Intel Corporation - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at https://www.aomedia.org/license/software-license. If the - * Alliance for Open Media Patent License 1.0 was not distributed with this - * source code in the PATENTS file, you can obtain it at - * https://www.aomedia.org/license/patent-license. - */ - -#ifndef AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_SSE4_1_H_ -#define AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_SSE4_1_H_ - -#include <smmintrin.h> - -#include "config/aom_config.h" - -#include "aom/aom_integer.h" -#include "aom_dsp/x86/mem_sse2.h" - -static inline __m128i load8bit_4x2_sse4_1(const void *const src, - const ptrdiff_t strideInByte) { - const __m128i s = _mm_cvtsi32_si128(loadu_int32(src)); - return _mm_insert_epi32(s, loadu_int32((uint8_t *)src + strideInByte), 1); -} - -static inline __m128i load_u8_4x2_sse4_1(const uint8_t *const src, - const ptrdiff_t stride) { - return load8bit_4x2_sse4_1(src, sizeof(*src) * stride); -} - -static inline __m128i load_u16_2x2_sse4_1(const uint16_t *const src, - const ptrdiff_t stride) { - return load8bit_4x2_sse4_1(src, sizeof(*src) * stride); -} - -#endif // AOM_THIRD_PARTY_SVT_AV1_EBMEMORY_SSE4_1_H_
diff --git a/third_party/SVT-AV1/LICENSE.md b/third_party/SVT-AV1/LICENSE.md deleted file mode 100644 index aff96d1..0000000 --- a/third_party/SVT-AV1/LICENSE.md +++ /dev/null
@@ -1,32 +0,0 @@ -BSD 3-Clause Clear License -The Clear BSD License - -Copyright (c) 2021, Alliance for Open Media - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted (subject to the limitations in the disclaimer below) -provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the distribution. - -3. Neither the name of the Alliance for Open Media nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT -OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/third_party/SVT-AV1/PATENTS.md b/third_party/SVT-AV1/PATENTS.md deleted file mode 100644 index 1de4dd7..0000000 --- a/third_party/SVT-AV1/PATENTS.md +++ /dev/null
@@ -1,107 +0,0 @@ -**Alliance for Open Media Patent License 1.0** - - 1. **License Terms.** - - **Patent License.** Subject to the terms and conditions of this License, each - Licensor, on behalf of itself and successors in interest and assigns, - grants Licensee a non-sublicensable, perpetual, worldwide, non-exclusive, - no-charge, royalty-free, irrevocable (except as expressly stated in this - License) patent license to its Necessary Claims to make, use, sell, offer - for sale, import or distribute any Implementation. - - **Conditions.** - - *Availability.* As a condition to the grant of rights to Licensee to make, - sell, offer for sale, import or distribute an Implementation under - Section 1.1, Licensee must make its Necessary Claims available under - this License, and must reproduce this License with any Implementation - as follows: - - a. For distribution in source code, by including this License in the - root directory of the source code with its Implementation. - - b. For distribution in any other form (including binary, object form, - and/or hardware description code (e.g., HDL, RTL, Gate Level Netlist, - GDSII, etc.)), by including this License in the documentation, legal - notices, and/or other written materials provided with the - Implementation. - - *Additional Conditions.* This license is directly from Licensor to - Licensee. Licensee acknowledges as a condition of benefiting from it - that no rights from Licensor are received from suppliers, distributors, - or otherwise in connection with this License. - - **Defensive Termination**. If any Licensee, its Affiliates, or its agents - initiates patent litigation or files, maintains, or voluntarily - participates in a lawsuit against another entity or any person asserting - that any Implementation infringes Necessary Claims, any patent licenses - granted under this License directly to the Licensee are immediately - terminated as of the date of the initiation of action unless 1) that suit - was in response to a corresponding suit regarding an Implementation first - brought against an initiating entity, or 2) that suit was brought to - enforce the terms of this License (including intervention in a third-party - action by a Licensee). - - **Disclaimers.** The Reference Implementation and Specification are provided - "AS IS" and without warranty. The entire risk as to implementing or - otherwise using the Reference Implementation or Specification is assumed - by the implementer and user. Licensor expressly disclaims any warranties - (express, implied, or otherwise), including implied warranties of - merchantability, non-infringement, fitness for a particular purpose, or - title, related to the material. IN NO EVENT WILL LICENSOR BE LIABLE TO - ANY OTHER PARTY FOR LOST PROFITS OR ANY FORM OF INDIRECT, SPECIAL, - INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER FROM ANY CAUSES OF - ACTION OF ANY KIND WITH RESPECT TO THIS LICENSE, WHETHER BASED ON BREACH - OF CONTRACT, TORT (INCLUDING NEGLIGENCE), OR OTHERWISE, AND WHETHER OR - NOT THE OTHER PARTRY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -2. **Definitions.** - - **Affiliate.** "Affiliate" means an entity that directly or indirectly - Controls, is Controlled by, or is under common Control of that party. - - **Control.** "Control" means direct or indirect control of more than 50% of - the voting power to elect directors of that corporation, or for any other - entity, the power to direct management of such entity. - - **Decoder.** "Decoder" means any decoder that conforms fully with all - non-optional portions of the Specification. - - **Encoder.** "Encoder" means any encoder that produces a bitstream that can - be decoded by a Decoder only to the extent it produces such a bitstream. - - **Final Deliverable.** "Final Deliverable" means the final version of a - deliverable approved by the Alliance for Open Media as a Final - Deliverable. - - **Implementation.** "Implementation" means any implementation, including the - Reference Implementation, that is an Encoder and/or a Decoder. An - Implementation also includes components of an Implementation only to the - extent they are used as part of an Implementation. - - **License.** "License" means this license. - - **Licensee.** "Licensee" means any person or entity who exercises patent - rights granted under this License. - - **Licensor.** "Licensor" means (i) any Licensee that makes, sells, offers - for sale, imports or distributes any Implementation, or (ii) a person - or entity that has a licensing obligation to the Implementation as a - result of its membership and/or participation in the Alliance for Open - Media working group that developed the Specification. - - **Necessary Claims.** "Necessary Claims" means all claims of patents or - patent applications, (a) that currently or at any time in the future, - are owned or controlled by the Licensor, and (b) (i) would be an - Essential Claim as defined by the W3C Policy as of February 5, 2004 - (https://www.w3.org/Consortium/Patent-Policy-20040205/#def-essential) - as if the Specification was a W3C Recommendation; or (ii) are infringed - by the Reference Implementation. - - **Reference Implementation.** "Reference Implementation" means an Encoder - and/or Decoder released by the Alliance for Open Media as a Final - Deliverable. - - **Specification.** "Specification" means the specification designated by - the Alliance for Open Media as a Final Deliverable for which this - License was issued.
diff --git a/third_party/SVT-AV1/README.libaom b/third_party/SVT-AV1/README.libaom deleted file mode 100644 index 069f7c1..0000000 --- a/third_party/SVT-AV1/README.libaom +++ /dev/null
@@ -1,20 +0,0 @@ -Name: SVT-AV1 -URL: https://gitlab.com/AOMediaCodec/SVT-AV1 -Version: N/A -Update Mechanism: Manual -Revision: 8ff99c90359330d2e807757c9425560bbc452ff3 -License: BSD-3-Clause-Clear -License File: LICENSE.md -Shipped in Chromium: no -Security Critical: yes - -Description: -Port the x86 intrinsics used for single reference convolve reconstructions. - -Local Changes: -Only ported the functions pertinent to single reference convolves. -All functions are made static inline to avoid function call overheads. -References to some arrays are changed to libaom version when applicable. -Some extra intrinsic functions are added to support missing block sizes. -Unaligned integer loads are changed to use functions from -aom_dsp/x86/mem_sse2.h.
diff --git a/third_party/SVT-AV1/convolve_2d_avx2.h b/third_party/SVT-AV1/convolve_2d_avx2.h deleted file mode 100644 index 64cd810..0000000 --- a/third_party/SVT-AV1/convolve_2d_avx2.h +++ /dev/null
@@ -1,1199 +0,0 @@ -/* - * Copyright (c) 2017, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef THIRD_PARTY_SVT_AV1_CONVOLVE_2D_AVX2_H_ -#define THIRD_PARTY_SVT_AV1_CONVOLVE_2D_AVX2_H_ - -#include "convolve_avx2.h" - -static void convolve_2d_sr_hor_2tap_avx2( - const uint8_t *const src, const int32_t src_stride, const int32_t w, - const int32_t h, const InterpFilterParams *const filter_params_x, - const int32_t subpel_x_q4, int16_t *const im_block) { - const uint8_t *src_ptr = src; - int32_t y = h; - int16_t *im = im_block; - - if (w <= 8) { - __m128i coeffs_128; - - prepare_half_coeffs_2tap_ssse3(filter_params_x, subpel_x_q4, &coeffs_128); - - if (w == 2) { - do { - const __m128i r = - x_convolve_2tap_2x2_sse4_1(src_ptr, src_stride, &coeffs_128); - xy_x_round_store_2x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 2; - y -= 2; - } while (y); - } else if (w == 4) { - do { - const __m128i r = - x_convolve_2tap_4x2_ssse3(src_ptr, src_stride, &coeffs_128); - xy_x_round_store_4x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 4; - y -= 2; - } while (y); - } else { - assert(w == 8); - - do { - __m128i r[2]; - - x_convolve_2tap_8x2_ssse3(src_ptr, src_stride, &coeffs_128, r); - xy_x_round_store_8x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 8; - y -= 2; - } while (y); - } - } else { - __m256i coeffs_256; - - prepare_half_coeffs_2tap_avx2(filter_params_x, subpel_x_q4, &coeffs_256); - - if (w == 16) { - do { - __m256i r[2]; - - x_convolve_2tap_16x2_avx2(src_ptr, src_stride, &coeffs_256, r); - xy_x_round_store_32_avx2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 16; - y -= 2; - } while (y); - } else if (w == 32) { - do { - xy_x_2tap_32_avx2(src_ptr, &coeffs_256, im); - src_ptr += src_stride; - im += 32; - } while (--y); - } else if (w == 64) { - do { - xy_x_2tap_32_avx2(src_ptr + 0 * 32, &coeffs_256, im + 0 * 32); - xy_x_2tap_32_avx2(src_ptr + 1 * 32, &coeffs_256, im + 1 * 32); - src_ptr += src_stride; - im += 64; - } while (--y); - } else { - assert(w == 128); - - do { - xy_x_2tap_32_avx2(src_ptr + 0 * 32, &coeffs_256, im + 0 * 32); - xy_x_2tap_32_avx2(src_ptr + 1 * 32, &coeffs_256, im + 1 * 32); - xy_x_2tap_32_avx2(src_ptr + 2 * 32, &coeffs_256, im + 2 * 32); - xy_x_2tap_32_avx2(src_ptr + 3 * 32, &coeffs_256, im + 3 * 32); - src_ptr += src_stride; - im += 128; - } while (--y); - } - } -} - -static void convolve_2d_sr_hor_4tap_ssse3( - const uint8_t *const src, const int32_t src_stride, const int32_t w, - const int32_t h, const InterpFilterParams *const filter_params_x, - const int32_t subpel_x_q4, int16_t *const im_block) { - const uint8_t *src_ptr = src - 1; - int32_t y = h; - int16_t *im = im_block; - - if (w <= 4) { - __m128i coeffs_128[2]; - - prepare_half_coeffs_4tap_ssse3(filter_params_x, subpel_x_q4, coeffs_128); - if (w == 2) { - do { - const __m128i r = - x_convolve_4tap_2x2_ssse3(src_ptr, src_stride, coeffs_128); - xy_x_round_store_2x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 2; - y -= 2; - } while (y); - } else if (w == 4) { - do { - const __m128i r = - x_convolve_4tap_4x2_ssse3(src_ptr, src_stride, coeffs_128); - xy_x_round_store_4x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 4; - y -= 2; - } while (y); - } - } else { - // TODO(chiyotsai@google.com): Add better optimization - __m256i coeffs_256[2], filt_256[2]; - - prepare_half_coeffs_4tap_avx2(filter_params_x, subpel_x_q4, coeffs_256); - filt_256[0] = _mm256_load_si256((__m256i const *)filt1_global_avx2); - filt_256[1] = _mm256_load_si256((__m256i const *)filt2_global_avx2); - - if (w == 8) { - do { - __m256i res = - x_convolve_4tap_8x2_avx2(src_ptr, src_stride, coeffs_256, filt_256); - xy_x_round_store_8x2_avx2(res, im); - - src_ptr += 2 * src_stride; - im += 2 * 8; - y -= 2; - } while (y); - } else if (w == 16) { - do { - __m256i r[2]; - - x_convolve_4tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, r); - xy_x_round_store_32_avx2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 16; - y -= 2; - } while (y); - } else if (w == 32) { - do { - xy_x_4tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - - src_ptr += src_stride; - im += 32; - } while (--y); - } else if (w == 64) { - do { - xy_x_4tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - xy_x_4tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, im + 32); - src_ptr += src_stride; - im += 64; - } while (--y); - } else { - assert(w == 128); - - do { - xy_x_4tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - xy_x_4tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, im + 32); - xy_x_4tap_32_avx2(src_ptr + 64, coeffs_256, filt_256, im + 64); - xy_x_4tap_32_avx2(src_ptr + 96, coeffs_256, filt_256, im + 96); - src_ptr += src_stride; - im += 128; - } while (--y); - } - } -} - -static void convolve_2d_sr_hor_6tap_avx2( - const uint8_t *const src, const int32_t src_stride, const int32_t w, - const int32_t h, const InterpFilterParams *const filter_params_x, - const int32_t subpel_x_q4, int16_t *const im_block) { - const uint8_t *src_ptr = src - 2; - int32_t y = h; - int16_t *im = im_block; - - if (w <= 4) { - __m128i coeffs_128[3]; - - prepare_half_coeffs_6tap_ssse3(filter_params_x, subpel_x_q4, coeffs_128); - if (w == 2) { - do { - const __m128i r = - x_convolve_6tap_2x2_ssse3(src_ptr, src_stride, coeffs_128); - xy_x_round_store_2x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 2; - y -= 2; - } while (y); - } else if (w == 4) { - do { - const __m128i r = - x_convolve_6tap_4x2_ssse3(src_ptr, src_stride, coeffs_128); - xy_x_round_store_4x2_sse2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 4; - y -= 2; - } while (y); - } - } else { - __m256i coeffs_256[3], filt_256[3]; - - filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx2); - filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx2); - filt_256[2] = _mm256_loadu_si256((__m256i const *)filt3_global_avx2); - - prepare_half_coeffs_6tap_avx2(filter_params_x, subpel_x_q4, coeffs_256); - - if (w == 8) { - do { - const __m256i res = - x_convolve_6tap_8x2_avx2(src_ptr, src_stride, coeffs_256, filt_256); - xy_x_round_store_8x2_avx2(res, im); - - src_ptr += 2 * src_stride; - im += 2 * 8; - y -= 2; - } while (y); - } else if (w == 16) { - do { - __m256i r[2]; - - x_convolve_6tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, r); - xy_x_round_store_32_avx2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 16; - y -= 2; - } while (y); - } else if (w == 32) { - do { - xy_x_6tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - src_ptr += src_stride; - im += 32; - } while (--y); - } else if (w == 64) { - do { - xy_x_6tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - xy_x_6tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, im + 32); - src_ptr += src_stride; - im += 64; - } while (--y); - } else { - assert(w == 128); - - do { - xy_x_6tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - xy_x_6tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, im + 32); - xy_x_6tap_32_avx2(src_ptr + 64, coeffs_256, filt_256, im + 64); - xy_x_6tap_32_avx2(src_ptr + 96, coeffs_256, filt_256, im + 96); - src_ptr += src_stride; - im += 128; - } while (--y); - } - } -} - -static void convolve_2d_sr_hor_8tap_avx2( - const uint8_t *const src, const int32_t src_stride, const int32_t w, - const int32_t h, const InterpFilterParams *const filter_params_x, - const int32_t subpel_x_q4, int16_t *const im_block) { - const uint8_t *src_ptr = src - 3; - int32_t y = h; - int16_t *im = im_block; - __m256i coeffs_256[4], filt_256[4]; - - filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx2); - filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx2); - filt_256[2] = _mm256_loadu_si256((__m256i const *)filt3_global_avx2); - filt_256[3] = _mm256_loadu_si256((__m256i const *)filt4_global_avx2); - - prepare_half_coeffs_8tap_avx2(filter_params_x, subpel_x_q4, coeffs_256); - - if (w == 8) { - do { - const __m256i res = - x_convolve_8tap_8x2_avx2(src_ptr, src_stride, coeffs_256, filt_256); - xy_x_round_store_8x2_avx2(res, im); - src_ptr += 2 * src_stride; - im += 2 * 8; - y -= 2; - } while (y); - } else if (w == 16) { - do { - __m256i r[2]; - - x_convolve_8tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, r); - xy_x_round_store_32_avx2(r, im); - src_ptr += 2 * src_stride; - im += 2 * 16; - y -= 2; - } while (y); - } else if (w == 32) { - do { - xy_x_8tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - src_ptr += src_stride; - im += 32; - } while (--y); - } else if (w == 64) { - do { - xy_x_8tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - xy_x_8tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, im + 32); - src_ptr += src_stride; - im += 64; - } while (--y); - } else { - assert(w == 128); - - do { - xy_x_8tap_32_avx2(src_ptr, coeffs_256, filt_256, im); - xy_x_8tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, im + 32); - xy_x_8tap_32_avx2(src_ptr + 64, coeffs_256, filt_256, im + 64); - xy_x_8tap_32_avx2(src_ptr + 96, coeffs_256, filt_256, im + 96); - src_ptr += src_stride; - im += 128; - } while (--y); - } -} - -static void convolve_2d_sr_ver_2tap_avx2( - const int16_t *const im_block, const int32_t w, const int32_t h, - const InterpFilterParams *const filter_params_y, const int32_t subpel_y_q4, - uint8_t *dst, const int32_t dst_stride) { - const int16_t *im = im_block; - int32_t y = h; - - if (w <= 4) { - __m128i coeffs_128; - - prepare_coeffs_2tap_sse2(filter_params_y, subpel_y_q4, &coeffs_128); - - if (w == 2) { - __m128i s_32[2]; - - s_32[0] = _mm_cvtsi32_si128(*(int32_t *)im); - - do { - const __m128i res = xy_y_convolve_2tap_2x2_sse2(im, s_32, &coeffs_128); - xy_y_round_store_2x2_sse2(res, dst, dst_stride); - im += 2 * 2; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m128i s_64[2], r[2]; - - assert(w == 4); - - s_64[0] = _mm_loadl_epi64((__m128i *)im); - - do { - xy_y_convolve_2tap_4x2_sse2(im, s_64, &coeffs_128, r); - r[0] = xy_y_round_sse2(r[0]); - r[1] = xy_y_round_sse2(r[1]); - const __m128i rr = _mm_packs_epi32(r[0], r[1]); - pack_store_4x2_sse2(rr, dst, dst_stride); - im += 2 * 4; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - __m256i coeffs_256; - - prepare_coeffs_2tap_avx2(filter_params_y, subpel_y_q4, &coeffs_256); - - if (w == 8) { - __m128i s_128[2]; - __m256i r[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)im); - - do { - xy_y_convolve_2tap_8x2_avx2(im, s_128, &coeffs_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - __m256i s_256[2], r[4]; - - s_256[0] = _mm256_loadu_si256((__m256i *)im); - - do { - xy_y_convolve_2tap_16x2_avx2(im, s_256, &coeffs_256, r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - __m256i s_256[2][2]; - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(im + 0 * 16)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(im + 1 * 16)); - - do { - xy_y_convolve_2tap_32_all_avx2(im + 32, s_256[0], s_256[1], &coeffs_256, - dst); - im += 2 * 32; - xy_y_convolve_2tap_32_all_avx2(im, s_256[1], s_256[0], &coeffs_256, - dst + dst_stride); - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 64) { - __m256i s_256[2][4]; - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(im + 0 * 16)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(im + 1 * 16)); - s_256[0][2] = _mm256_loadu_si256((__m256i *)(im + 2 * 16)); - s_256[0][3] = _mm256_loadu_si256((__m256i *)(im + 3 * 16)); - - do { - xy_y_convolve_2tap_32_all_avx2(im + 64, s_256[0] + 0, s_256[1] + 0, - &coeffs_256, dst); - xy_y_convolve_2tap_32_all_avx2(im + 96, s_256[0] + 2, s_256[1] + 2, - &coeffs_256, dst + 32); - im += 2 * 64; - xy_y_convolve_2tap_32_all_avx2(im, s_256[1] + 0, s_256[0] + 0, - &coeffs_256, dst + dst_stride); - xy_y_convolve_2tap_32_all_avx2(im + 32, s_256[1] + 2, s_256[0] + 2, - &coeffs_256, dst + dst_stride + 32); - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i s_256[2][8]; - - assert(w == 128); - - load_16bit_8rows_avx2(im, 16, s_256[0]); - - do { - xy_y_convolve_2tap_32_all_avx2(im + 128, s_256[0] + 0, s_256[1] + 0, - &coeffs_256, dst); - xy_y_convolve_2tap_32_all_avx2(im + 160, s_256[0] + 2, s_256[1] + 2, - &coeffs_256, dst + 1 * 32); - xy_y_convolve_2tap_32_all_avx2(im + 192, s_256[0] + 4, s_256[1] + 4, - &coeffs_256, dst + 2 * 32); - xy_y_convolve_2tap_32_all_avx2(im + 224, s_256[0] + 6, s_256[1] + 6, - &coeffs_256, dst + 3 * 32); - im += 2 * 128; - xy_y_convolve_2tap_32_all_avx2(im, s_256[1] + 0, s_256[0] + 0, - &coeffs_256, dst + dst_stride); - xy_y_convolve_2tap_32_all_avx2(im + 32, s_256[1] + 2, s_256[0] + 2, - &coeffs_256, dst + dst_stride + 1 * 32); - xy_y_convolve_2tap_32_all_avx2(im + 64, s_256[1] + 4, s_256[0] + 4, - &coeffs_256, dst + dst_stride + 2 * 32); - xy_y_convolve_2tap_32_all_avx2(im + 96, s_256[1] + 6, s_256[0] + 6, - &coeffs_256, dst + dst_stride + 3 * 32); - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } -} - -static void convolve_2d_sr_ver_2tap_half_avx2( - const int16_t *const im_block, const int32_t w, const int32_t h, - const InterpFilterParams *const filter_params_y, const int32_t subpel_y_q4, - uint8_t *dst, const int32_t dst_stride) { - const int16_t *im = im_block; - int32_t y = h; - - (void)filter_params_y; - (void)subpel_y_q4; - - if (w == 2) { - __m128i s_32[2]; - - s_32[0] = _mm_cvtsi32_si128(*(int32_t *)im); - - do { - const __m128i res = xy_y_convolve_2tap_2x2_half_pel_sse2(im, s_32); - const __m128i r = xy_y_round_half_pel_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - im += 2 * 2; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 4) { - __m128i s_64[2]; - - s_64[0] = _mm_loadl_epi64((__m128i *)im); - - do { - const __m128i res = xy_y_convolve_2tap_4x2_half_pel_sse2(im, s_64); - const __m128i r = xy_y_round_half_pel_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - im += 2 * 4; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 8) { - __m128i s_128[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)im); - - do { - const __m256i res = xy_y_convolve_2tap_8x2_half_pel_avx2(im, s_128); - const __m256i r = xy_y_round_half_pel_avx2(res); - pack_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - __m256i s_256[2], r[2]; - - s_256[0] = _mm256_loadu_si256((__m256i *)im); - - do { - xy_y_convolve_2tap_16x2_half_pel_avx2(im, s_256, r); - r[0] = xy_y_round_half_pel_avx2(r[0]); - r[1] = xy_y_round_half_pel_avx2(r[1]); - xy_y_pack_store_16x2_avx2(r[0], r[1], dst, dst_stride); - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - __m256i s_256[2][2]; - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(im + 0 * 16)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(im + 1 * 16)); - - do { - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 32, s_256[0], s_256[1], dst); - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 2 * 32, s_256[1], s_256[0], - dst + dst_stride); - im += 2 * 32; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 64) { - __m256i s_256[2][4]; - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(im + 0 * 16)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(im + 1 * 16)); - s_256[0][2] = _mm256_loadu_si256((__m256i *)(im + 2 * 16)); - s_256[0][3] = _mm256_loadu_si256((__m256i *)(im + 3 * 16)); - - do { - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 64, s_256[0] + 0, - s_256[1] + 0, dst); - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 96, s_256[0] + 2, - s_256[1] + 2, dst + 32); - im += 2 * 64; - xy_y_convolve_2tap_half_pel_32_all_avx2(im, s_256[1] + 0, s_256[0] + 0, - dst + dst_stride); - xy_y_convolve_2tap_half_pel_32_all_avx2( - im + 32, s_256[1] + 2, s_256[0] + 2, dst + dst_stride + 32); - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i s_256[2][8]; - - assert(w == 128); - - load_16bit_8rows_avx2(im, 16, s_256[0]); - - do { - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 128, s_256[0] + 0, - s_256[1] + 0, dst); - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 160, s_256[0] + 2, - s_256[1] + 2, dst + 1 * 32); - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 192, s_256[0] + 4, - s_256[1] + 4, dst + 2 * 32); - xy_y_convolve_2tap_half_pel_32_all_avx2(im + 224, s_256[0] + 6, - s_256[1] + 6, dst + 3 * 32); - im += 2 * 128; - xy_y_convolve_2tap_half_pel_32_all_avx2(im, s_256[1] + 0, s_256[0] + 0, - dst + dst_stride); - xy_y_convolve_2tap_half_pel_32_all_avx2( - im + 32, s_256[1] + 2, s_256[0] + 2, dst + dst_stride + 1 * 32); - xy_y_convolve_2tap_half_pel_32_all_avx2( - im + 64, s_256[1] + 4, s_256[0] + 4, dst + dst_stride + 2 * 32); - xy_y_convolve_2tap_half_pel_32_all_avx2( - im + 96, s_256[1] + 6, s_256[0] + 6, dst + dst_stride + 3 * 32); - dst += 2 * dst_stride; - y -= 2; - } while (y); - } -} - -static void convolve_2d_sr_ver_4tap_avx2( - const int16_t *const im_block, const int32_t w, const int32_t h, - const InterpFilterParams *const filter_params_y, const int32_t subpel_y_q4, - uint8_t *dst, const int32_t dst_stride) { - const int16_t *im = im_block; - int32_t y = h; - - if (w == 2) { - __m128i coeffs_128[2], s_32[4], ss_128[2]; - - prepare_coeffs_4tap_sse2(filter_params_y, subpel_y_q4, coeffs_128); - - s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(im + 0 * 2)); - s_32[1] = _mm_cvtsi32_si128(*(int32_t *)(im + 1 * 2)); - s_32[2] = _mm_cvtsi32_si128(*(int32_t *)(im + 2 * 2)); - - const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]); - const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]); - - ss_128[0] = _mm_unpacklo_epi16(src01, src12); - - do { - const __m128i res = - xy_y_convolve_4tap_2x2_sse2(im, s_32, ss_128, coeffs_128); - xy_y_round_store_2x2_sse2(res, dst, dst_stride); - im += 2 * 2; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i coeffs_256[2]; - - prepare_coeffs_4tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 4) { - __m128i s_64[4]; - __m256i s_256[2], ss_256[2]; - - s_64[0] = _mm_loadl_epi64((__m128i *)(im + 0 * 4)); - s_64[1] = _mm_loadl_epi64((__m128i *)(im + 1 * 4)); - s_64[2] = _mm_loadl_epi64((__m128i *)(im + 2 * 4)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - s_256[0] = _mm256_setr_m128i(s_64[0], s_64[1]); - s_256[1] = _mm256_setr_m128i(s_64[1], s_64[2]); - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - - do { - const __m256i res = - xy_y_convolve_4tap_4x2_avx2(im, s_64, ss_256, coeffs_256); - xy_y_round_store_4x2_avx2(res, dst, dst_stride); - im += 2 * 4; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 8) { - __m256i s_256[4], r[2]; - - s_256[0] = _mm256_loadu_si256((__m256i *)(im + 0 * 8)); - s_256[1] = _mm256_loadu_si256((__m256i *)(im + 1 * 8)); - - if (subpel_y_q4 != 8) { - __m256i ss_256[4]; - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[2] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - - do { - xy_y_convolve_4tap_8x2_avx2(im, ss_256, coeffs_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - do { - xy_y_convolve_4tap_8x2_half_pel_avx2(im, coeffs_256, s_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else if (w == 16) { - __m256i s_256[5]; - - s_256[0] = _mm256_loadu_si256((__m256i *)(im + 0 * 16)); - s_256[1] = _mm256_loadu_si256((__m256i *)(im + 1 * 16)); - s_256[2] = _mm256_loadu_si256((__m256i *)(im + 2 * 16)); - - if (subpel_y_q4 != 8) { - __m256i ss_256[4], tt_256[4], r[4]; - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[2] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - - tt_256[0] = _mm256_unpacklo_epi16(s_256[1], s_256[2]); - tt_256[2] = _mm256_unpackhi_epi16(s_256[1], s_256[2]); - - do { - xy_y_convolve_4tap_16x2_avx2(im, s_256, ss_256, tt_256, coeffs_256, - r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i r[4]; - - do { - xy_y_convolve_4tap_16x2_half_pelavx2(im, s_256, coeffs_256, r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - /*It's a special condition for OBMC. A/c to Av1 spec 4-tap won't - support for width(w)>16, but for OBMC while predicting above block - it reduces size block to Wx(h/2), for example, if above block size - is 32x8, we get block size as 32x4 for OBMC.*/ - int32_t x = 0; - - assert(!(w % 32)); - - __m256i s_256[2][4], ss_256[2][4], tt_256[2][4], r0[4], r1[4]; - do { - const int16_t *s = im + x; - uint8_t *d = dst + x; - - loadu_unpack_16bit_3rows_avx2(s, w, s_256[0], ss_256[0], tt_256[0]); - loadu_unpack_16bit_3rows_avx2(s + 16, w, s_256[1], ss_256[1], - tt_256[1]); - - y = h; - do { - xy_y_convolve_4tap_32x2_avx2(s, w, s_256[0], ss_256[0], tt_256[0], - coeffs_256, r0); - xy_y_convolve_4tap_32x2_avx2(s + 16, w, s_256[1], ss_256[1], - tt_256[1], coeffs_256, r1); - - xy_y_round_store_32_avx2(r0 + 0, r1 + 0, d); - xy_y_round_store_32_avx2(r0 + 2, r1 + 2, d + dst_stride); - - s += 2 * w; - d += 2 * dst_stride; - y -= 2; - } while (y); - - x += 32; - } while (x < w); - } - } -} - -static void convolve_2d_sr_ver_6tap_avx2( - const int16_t *const im_block, const int32_t w, const int32_t h, - const InterpFilterParams *const filter_params_y, const int32_t subpel_y_q4, - uint8_t *dst, const int32_t dst_stride) { - const int16_t *im = im_block; - int32_t y; - - if (w == 2) { - __m128i coeffs_128[3], s_32[6], ss_128[3]; - - prepare_coeffs_6tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128); - - s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(im + 0 * 2)); - s_32[1] = _mm_cvtsi32_si128(*(int32_t *)(im + 1 * 2)); - s_32[2] = _mm_cvtsi32_si128(*(int32_t *)(im + 2 * 2)); - s_32[3] = _mm_cvtsi32_si128(*(int32_t *)(im + 3 * 2)); - s_32[4] = _mm_cvtsi32_si128(*(int32_t *)(im + 4 * 2)); - - const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]); - const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]); - const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]); - const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[4]); - - ss_128[0] = _mm_unpacklo_epi16(src01, src12); - ss_128[1] = _mm_unpacklo_epi16(src23, src34); - - y = h; - do { - const __m128i res = - xy_y_convolve_6tap_2x2_sse2(im, s_32, ss_128, coeffs_128); - xy_y_round_store_2x2_sse2(res, dst, dst_stride); - im += 2 * 2; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i coeffs_256[3]; - - prepare_coeffs_6tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 4) { - __m128i s_64[6]; - __m256i s_256[6], ss_256[3]; - - s_64[0] = _mm_loadl_epi64((__m128i *)(im + 0 * 4)); - s_64[1] = _mm_loadl_epi64((__m128i *)(im + 1 * 4)); - s_64[2] = _mm_loadl_epi64((__m128i *)(im + 2 * 4)); - s_64[3] = _mm_loadl_epi64((__m128i *)(im + 3 * 4)); - s_64[4] = _mm_loadl_epi64((__m128i *)(im + 4 * 4)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - s_256[0] = _mm256_setr_m128i(s_64[0], s_64[1]); - s_256[1] = _mm256_setr_m128i(s_64[1], s_64[2]); - s_256[2] = _mm256_setr_m128i(s_64[2], s_64[3]); - s_256[3] = _mm256_setr_m128i(s_64[3], s_64[4]); - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - - y = h; - do { - const __m256i res = - xy_y_convolve_6tap_4x2_avx2(im, s_64, ss_256, coeffs_256); - xy_y_round_store_4x2_avx2(res, dst, dst_stride); - im += 2 * 4; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 8) { - __m256i s_256[6], r[2]; - - s_256[0] = _mm256_loadu_si256((__m256i *)(im + 0 * 8)); - s_256[1] = _mm256_loadu_si256((__m256i *)(im + 1 * 8)); - s_256[2] = _mm256_loadu_si256((__m256i *)(im + 2 * 8)); - s_256[3] = _mm256_loadu_si256((__m256i *)(im + 3 * 8)); - y = h; - - if (subpel_y_q4 != 8) { - __m256i ss_256[6]; - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - - ss_256[3] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - ss_256[4] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - - do { - xy_y_convolve_6tap_8x2_avx2(im, ss_256, coeffs_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - do { - xy_y_convolve_6tap_8x2_half_pel_avx2(im, coeffs_256, s_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else if (w == 16) { - __m256i s_256[6]; - - s_256[0] = _mm256_loadu_si256((__m256i *)(im + 0 * 16)); - s_256[1] = _mm256_loadu_si256((__m256i *)(im + 1 * 16)); - s_256[2] = _mm256_loadu_si256((__m256i *)(im + 2 * 16)); - s_256[3] = _mm256_loadu_si256((__m256i *)(im + 3 * 16)); - s_256[4] = _mm256_loadu_si256((__m256i *)(im + 4 * 16)); - y = h; - - if (subpel_y_q4 != 8) { - __m256i ss_256[6], tt_256[6], r[4]; - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - ss_256[4] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - - tt_256[0] = _mm256_unpacklo_epi16(s_256[1], s_256[2]); - tt_256[1] = _mm256_unpacklo_epi16(s_256[3], s_256[4]); - tt_256[3] = _mm256_unpackhi_epi16(s_256[1], s_256[2]); - tt_256[4] = _mm256_unpackhi_epi16(s_256[3], s_256[4]); - - do { - xy_y_convolve_6tap_16x2_avx2(im, 16, s_256, ss_256, tt_256, - coeffs_256, r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i ss_256[4], r[4]; - - do { - xy_y_convolve_6tap_16x2_half_pel_avx2(im, 16, s_256, ss_256, - coeffs_256, r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - int32_t x = 0; - - assert(!(w % 32)); - - __m256i s_256[2][6], ss_256[2][6], tt_256[2][6], r0[4], r1[4]; - - do { - const int16_t *s = im + x; - uint8_t *d = dst + x; - - loadu_unpack_16bit_5rows_avx2(s, w, s_256[0], ss_256[0], tt_256[0]); - loadu_unpack_16bit_5rows_avx2(s + 16, w, s_256[1], ss_256[1], - tt_256[1]); - - y = h; - do { - xy_y_convolve_6tap_16x2_avx2(s, w, s_256[0], ss_256[0], tt_256[0], - coeffs_256, r0); - xy_y_convolve_6tap_16x2_avx2(s + 16, w, s_256[1], ss_256[1], - tt_256[1], coeffs_256, r1); - - xy_y_round_store_32_avx2(r0 + 0, r1 + 0, d); - xy_y_round_store_32_avx2(r0 + 2, r1 + 2, d + dst_stride); - - s += 2 * w; - d += 2 * dst_stride; - y -= 2; - } while (y); - - x += 32; - } while (x < w); - } - } -} - -static void convolve_2d_sr_ver_8tap_avx2( - const int16_t *const im_block, const int32_t w, const int32_t h, - const InterpFilterParams *const filter_params_y, const int32_t subpel_y_q4, - uint8_t *dst, const int32_t dst_stride) { - const int16_t *im = im_block; - int32_t y; - - if (w == 2) { - __m128i coeffs_128[4], s_32[8], ss_128[4]; - - prepare_coeffs_8tap_sse2(filter_params_y, subpel_y_q4, coeffs_128); - - s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(im + 0 * 2)); - s_32[1] = _mm_cvtsi32_si128(*(int32_t *)(im + 1 * 2)); - s_32[2] = _mm_cvtsi32_si128(*(int32_t *)(im + 2 * 2)); - s_32[3] = _mm_cvtsi32_si128(*(int32_t *)(im + 3 * 2)); - s_32[4] = _mm_cvtsi32_si128(*(int32_t *)(im + 4 * 2)); - s_32[5] = _mm_cvtsi32_si128(*(int32_t *)(im + 5 * 2)); - s_32[6] = _mm_cvtsi32_si128(*(int32_t *)(im + 6 * 2)); - - const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]); - const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]); - const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]); - const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[4]); - const __m128i src45 = _mm_unpacklo_epi32(s_32[4], s_32[5]); - const __m128i src56 = _mm_unpacklo_epi32(s_32[5], s_32[6]); - - ss_128[0] = _mm_unpacklo_epi16(src01, src12); - ss_128[1] = _mm_unpacklo_epi16(src23, src34); - ss_128[2] = _mm_unpacklo_epi16(src45, src56); - - y = h; - do { - const __m128i res = - xy_y_convolve_8tap_2x2_sse2(im, s_32, ss_128, coeffs_128); - xy_y_round_store_2x2_sse2(res, dst, dst_stride); - im += 2 * 2; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i coeffs_256[4]; - - prepare_coeffs_8tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 4) { - __m128i s_64[8]; - __m256i s_256[8], ss_256[4]; - - s_64[0] = _mm_loadl_epi64((__m128i *)(im + 0 * 4)); - s_64[1] = _mm_loadl_epi64((__m128i *)(im + 1 * 4)); - s_64[2] = _mm_loadl_epi64((__m128i *)(im + 2 * 4)); - s_64[3] = _mm_loadl_epi64((__m128i *)(im + 3 * 4)); - s_64[4] = _mm_loadl_epi64((__m128i *)(im + 4 * 4)); - s_64[5] = _mm_loadl_epi64((__m128i *)(im + 5 * 4)); - s_64[6] = _mm_loadl_epi64((__m128i *)(im + 6 * 4)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - s_256[0] = _mm256_setr_m128i(s_64[0], s_64[1]); - s_256[1] = _mm256_setr_m128i(s_64[1], s_64[2]); - s_256[2] = _mm256_setr_m128i(s_64[2], s_64[3]); - s_256[3] = _mm256_setr_m128i(s_64[3], s_64[4]); - s_256[4] = _mm256_setr_m128i(s_64[4], s_64[5]); - s_256[5] = _mm256_setr_m128i(s_64[5], s_64[6]); - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[2] = _mm256_unpacklo_epi16(s_256[4], s_256[5]); - - y = h; - do { - const __m256i res = - xy_y_convolve_8tap_4x2_avx2(im, s_64, ss_256, coeffs_256); - xy_y_round_store_4x2_avx2(res, dst, dst_stride); - im += 2 * 4; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 8) { - __m256i s_256[8], r[2]; - - s_256[0] = _mm256_loadu_si256((__m256i *)(im + 0 * 8)); - s_256[1] = _mm256_loadu_si256((__m256i *)(im + 1 * 8)); - s_256[2] = _mm256_loadu_si256((__m256i *)(im + 2 * 8)); - s_256[3] = _mm256_loadu_si256((__m256i *)(im + 3 * 8)); - s_256[4] = _mm256_loadu_si256((__m256i *)(im + 4 * 8)); - s_256[5] = _mm256_loadu_si256((__m256i *)(im + 5 * 8)); - y = h; - - if (subpel_y_q4 != 8) { - __m256i ss_256[8]; - - convolve_8tap_unpack_avx2(s_256, ss_256); - - do { - xy_y_convolve_8tap_8x2_avx2(im, ss_256, coeffs_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - do { - xy_y_convolve_8tap_8x2_half_pel_avx2(im, coeffs_256, s_256, r); - xy_y_round_store_8x2_avx2(r, dst, dst_stride); - im += 2 * 8; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else if (w == 16) { - __m256i s_256[8], r[4]; - - load_16bit_7rows_avx2(im, 16, s_256); - y = h; - - if (subpel_y_q4 != 8) { - __m256i ss_256[8], tt_256[8]; - - convolve_8tap_unpack_avx2(s_256, ss_256); - convolve_8tap_unpack_avx2(s_256 + 1, tt_256); - - do { - xy_y_convolve_8tap_16x2_avx2(im, 16, coeffs_256, s_256, ss_256, - tt_256, r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - do { - xy_y_convolve_8tap_16x2_half_pel_avx2(im, 16, coeffs_256, s_256, r); - xy_y_round_store_16x2_avx2(r, dst, dst_stride); - - im += 2 * 16; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - int32_t x = 0; - __m256i s_256[2][8], r0[4], r1[4]; - - assert(!(w % 32)); - - __m256i ss_256[2][8], tt_256[2][8]; - - do { - const int16_t *s = im + x; - uint8_t *d = dst + x; - - load_16bit_7rows_avx2(s, w, s_256[0]); - convolve_8tap_unpack_avx2(s_256[0], ss_256[0]); - convolve_8tap_unpack_avx2(s_256[0] + 1, tt_256[0]); - - load_16bit_7rows_avx2(s + 16, w, s_256[1]); - convolve_8tap_unpack_avx2(s_256[1], ss_256[1]); - convolve_8tap_unpack_avx2(s_256[1] + 1, tt_256[1]); - - y = h; - do { - xy_y_convolve_8tap_16x2_avx2(s, w, coeffs_256, s_256[0], ss_256[0], - tt_256[0], r0); - xy_y_convolve_8tap_16x2_avx2(s + 16, w, coeffs_256, s_256[1], - ss_256[1], tt_256[1], r1); - xy_y_round_store_32_avx2(r0 + 0, r1 + 0, d); - xy_y_round_store_32_avx2(r0 + 2, r1 + 2, d + dst_stride); - - s += 2 * w; - d += 2 * dst_stride; - y -= 2; - } while (y); - - x += 32; - } while (x < w); - } - } -} - -typedef void (*Convolve2dSrHorTapFunc)( - const uint8_t *const src, const int32_t src_stride, const int32_t w, - const int32_t h, const InterpFilterParams *const filter_params_x, - const int32_t subpel_x_q4, int16_t *const im_block); - -typedef void (*Convolve2dSrVerTapFunc)( - const int16_t *const im_block, const int32_t w, const int32_t h, - const InterpFilterParams *const filter_params_y, const int32_t subpel_y_q4, - uint8_t *dst, const int32_t dst_stride); - -static AOM_FORCE_INLINE void av1_convolve_2d_sr_specialized_avx2( - const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, - int32_t w, int32_t h, const InterpFilterParams *filter_params_x, - const InterpFilterParams *filter_params_y, const int32_t subpel_x_q4, - const int32_t subpel_y_q4, ConvolveParams *conv_params) { - static const Convolve2dSrHorTapFunc - convolve_2d_sr_hor_tap_func_table[MAX_FILTER_TAP + 1] = { - NULL, - NULL, - convolve_2d_sr_hor_2tap_avx2, - NULL, - convolve_2d_sr_hor_4tap_ssse3, - NULL, - convolve_2d_sr_hor_6tap_avx2, - NULL, - convolve_2d_sr_hor_8tap_avx2 - }; - static const Convolve2dSrVerTapFunc - convolve_2d_sr_ver_tap_func_table[MAX_FILTER_TAP + 1] = { - NULL, - convolve_2d_sr_ver_2tap_half_avx2, - convolve_2d_sr_ver_2tap_avx2, - convolve_2d_sr_ver_4tap_avx2, - convolve_2d_sr_ver_4tap_avx2, - convolve_2d_sr_ver_6tap_avx2, - convolve_2d_sr_ver_6tap_avx2, - convolve_2d_sr_ver_8tap_avx2, - convolve_2d_sr_ver_8tap_avx2 - }; - const int32_t tap_x = get_filter_tap(filter_params_x, subpel_x_q4); - const int32_t tap_y = get_filter_tap(filter_params_y, subpel_y_q4); - - assert(tap_x != 12 && tap_y != 12); - - const uint8_t *src_ptr = src - ((tap_y >> 1) - 1) * src_stride; - // Note: im_block is 8-pixel interlaced for width 32 and up, to avoid data - // permutation. - DECLARE_ALIGNED(32, int16_t, - im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]); - - (void)conv_params; - - assert(conv_params->round_0 == 3); - assert(conv_params->round_1 == 11); - - // horizontal filter - int32_t hh = h + tap_y; - assert(!(hh % 2)); - - convolve_2d_sr_hor_tap_func_table[tap_x]( - src_ptr, src_stride, w, hh, filter_params_x, subpel_x_q4, im_block); - - // vertical filter - convolve_2d_sr_ver_tap_func_table[tap_y - (subpel_y_q4 == 8)]( - im_block, w, h, filter_params_y, subpel_y_q4, dst, dst_stride); -} - -#endif // THIRD_PARTY_SVT_AV1_CONVOLVE_2D_AVX2_H_
diff --git a/third_party/SVT-AV1/convolve_avx2.h b/third_party/SVT-AV1/convolve_avx2.h deleted file mode 100644 index 0e7e436..0000000 --- a/third_party/SVT-AV1/convolve_avx2.h +++ /dev/null
@@ -1,3335 +0,0 @@ -/* - * Copyright (c) 2018, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef THIRD_PARTY_SVT_AV1_CONVOLVE_AVX2_H_ -#define THIRD_PARTY_SVT_AV1_CONVOLVE_AVX2_H_ - -#include "EbMemory_AVX2.h" -#include "EbMemory_SSE4_1.h" -#include "synonyms.h" - -#include "aom_dsp/aom_filter.h" -#include "aom_dsp/x86/convolve_avx2.h" -#include "aom_dsp/x86/mem_sse2.h" - -static inline void populate_coeffs_4tap_avx2(const __m128i coeffs_128, - __m256i coeffs[2]) { - const __m256i coeffs_256 = _mm256_broadcastsi128_si256(coeffs_128); - - // coeffs 2 3 2 3 2 3 2 3 - coeffs[0] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0604u)); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[1] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0a08u)); -} - -static inline void populate_coeffs_6tap_avx2(const __m128i coeffs_128, - __m256i coeffs[3]) { - const __m256i coeffs_256 = _mm256_broadcastsi128_si256(coeffs_128); - - // coeffs 1 2 1 2 1 2 1 2 - coeffs[0] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0402u)); - // coeffs 3 4 3 4 3 4 3 4 - coeffs[1] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0806u)); - // coeffs 5 6 5 6 5 6 5 6 - coeffs[2] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0C0Au)); -} - -static inline void populate_coeffs_8tap_avx2(const __m128i coeffs_128, - __m256i coeffs[4]) { - const __m256i coeffs_256 = _mm256_broadcastsi128_si256(coeffs_128); - - // coeffs 0 1 0 1 0 1 0 1 - coeffs[0] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0200u)); - // coeffs 2 3 2 3 2 3 2 3 - coeffs[1] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0604u)); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[2] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0a08u)); - // coeffs 6 7 6 7 6 7 6 7 - coeffs[3] = _mm256_shuffle_epi8(coeffs_256, _mm256_set1_epi16(0x0e0cu)); -} - -static inline void prepare_half_coeffs_2tap_ssse3( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [1] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_cvtsi32_si128(loadu_int32(filter + 3)); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - - // coeffs 3 4 3 4 3 4 3 4 - *coeffs = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0200u)); -} - -static inline void prepare_half_coeffs_4tap_ssse3( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [2] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - - // coeffs 2 3 2 3 2 3 2 3 - coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0604u)); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[1] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0a08u)); -} - -static inline void prepare_half_coeffs_6tap_ssse3( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [3] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - - // coeffs 1 2 1 2 1 2 1 2 - coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0402u)); - // coeffs 3 4 3 4 3 4 3 4 - coeffs[1] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0806u)); - // coeffs 5 6 5 6 5 6 5 6 - coeffs[2] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0C0Au)); -} - -static inline void prepare_half_coeffs_8tap_ssse3( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [4] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - - // coeffs 0 1 0 1 0 1 0 1 - coeffs[0] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0200u)); - // coeffs 2 3 2 3 2 3 2 3 - coeffs[1] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0604u)); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[2] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0a08u)); - // coeffs 6 7 6 7 6 7 6 7 - coeffs[3] = _mm_shuffle_epi8(coeffs_1, _mm_set1_epi16(0x0e0cu)); -} - -static inline void prepare_half_coeffs_2tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [1] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_cvtsi32_si128(loadu_int32(filter + 3)); - const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - - const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); - - // coeffs 3 4 3 4 3 4 3 4 - *coeffs = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0200u)); -} - -static inline void prepare_half_coeffs_4tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [2] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - populate_coeffs_4tap_avx2(coeffs_1, coeffs); -} - -static inline void prepare_half_coeffs_6tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [3] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - populate_coeffs_6tap_avx2(coeffs_1, coeffs); -} - -static inline void prepare_half_coeffs_8tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [4] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - - // right shift all filter co-efficients by 1 to reduce the bits required. - // This extra right shift will be taken care of at the end while rounding - // the result. - // Since all filter co-efficients are even, this change will not affect the - // end result - assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), - _mm_set1_epi16((short)0xffff))); - const __m128i coeffs_1 = _mm_srai_epi16(coeffs_8, 1); - populate_coeffs_8tap_avx2(coeffs_1, coeffs); -} - -static inline void prepare_coeffs_2tap_sse2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [1] */) { - const int16_t *filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - - const __m128i coeff = _mm_cvtsi32_si128(loadu_int32(filter + 3)); - - // coeffs 3 4 3 4 3 4 3 4 - coeffs[0] = _mm_shuffle_epi32(coeff, 0x00); -} - -static inline void prepare_coeffs_4tap_sse2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [2] */) { - const int16_t *filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - - const __m128i coeff = _mm_loadu_si128((__m128i *)filter); - - // coeffs 2 3 2 3 2 3 2 3 - coeffs[0] = _mm_shuffle_epi32(coeff, 0x55); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[1] = _mm_shuffle_epi32(coeff, 0xaa); -} - -static inline void prepare_coeffs_6tap_ssse3( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [3] */) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeff = _mm_loadu_si128((__m128i *)filter); - - // coeffs 1 2 1 2 1 2 1 2 - coeffs[0] = _mm_shuffle_epi8(coeff, _mm_set1_epi32(0x05040302u)); - // coeffs 3 4 3 4 3 4 3 4 - coeffs[1] = _mm_shuffle_epi8(coeff, _mm_set1_epi32(0x09080706u)); - // coeffs 5 6 5 6 5 6 5 6 - coeffs[2] = _mm_shuffle_epi8(coeff, _mm_set1_epi32(0x0D0C0B0Au)); -} - -static inline void prepare_coeffs_8tap_sse2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m128i *const coeffs /* [4] */) { - const int16_t *filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - - const __m128i coeff = _mm_loadu_si128((__m128i *)filter); - - // coeffs 0 1 0 1 0 1 0 1 - coeffs[0] = _mm_shuffle_epi32(coeff, 0x00); - // coeffs 2 3 2 3 2 3 2 3 - coeffs[1] = _mm_shuffle_epi32(coeff, 0x55); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[2] = _mm_shuffle_epi32(coeff, 0xaa); - // coeffs 6 7 6 7 6 7 6 7 - coeffs[3] = _mm_shuffle_epi32(coeff, 0xff); -} - -static inline void prepare_coeffs_2tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [1] */) { - const int16_t *filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - - const __m128i coeff_8 = _mm_cvtsi32_si128(loadu_int32(filter + 3)); - const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); - - // coeffs 3 4 3 4 3 4 3 4 - coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); -} - -static inline void prepare_coeffs_4tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [2] */) { - const int16_t *filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - - const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); - const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); - - // coeffs 2 3 2 3 2 3 2 3 - coeffs[0] = _mm256_shuffle_epi32(coeff, 0x55); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[1] = _mm256_shuffle_epi32(coeff, 0xaa); -} - -static inline void prepare_coeffs_6tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [3]*/) { - const int16_t *const filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); - const __m256i coeff = _mm256_broadcastsi128_si256(coeffs_8); - - // coeffs 1 2 1 2 1 2 1 2 - coeffs[0] = _mm256_shuffle_epi8(coeff, _mm256_set1_epi32(0x05040302u)); - // coeffs 3 4 3 4 3 4 3 4 - coeffs[1] = _mm256_shuffle_epi8(coeff, _mm256_set1_epi32(0x09080706u)); - // coeffs 5 6 5 6 5 6 5 6 - coeffs[2] = _mm256_shuffle_epi8(coeff, _mm256_set1_epi32(0x0D0C0B0Au)); -} - -static inline void prepare_coeffs_8tap_avx2( - const InterpFilterParams *const filter_params, const int32_t subpel_q4, - __m256i *const coeffs /* [4] */) { - const int16_t *filter = av1_get_interp_filter_subpel_kernel( - filter_params, subpel_q4 & SUBPEL_MASK); - - const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); - const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); - - // coeffs 0 1 0 1 0 1 0 1 - coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); - // coeffs 2 3 2 3 2 3 2 3 - coeffs[1] = _mm256_shuffle_epi32(coeff, 0x55); - // coeffs 4 5 4 5 4 5 4 5 - coeffs[2] = _mm256_shuffle_epi32(coeff, 0xaa); - // coeffs 6 7 6 7 6 7 6 7 - coeffs[3] = _mm256_shuffle_epi32(coeff, 0xff); -} - -static inline void load_16bit_5rows_avx2(const int16_t *const src, - const ptrdiff_t stride, - __m256i dst[5]) { - dst[0] = _mm256_loadu_si256((__m256i *)(src + 0 * stride)); - dst[1] = _mm256_loadu_si256((__m256i *)(src + 1 * stride)); - dst[2] = _mm256_loadu_si256((__m256i *)(src + 2 * stride)); - dst[3] = _mm256_loadu_si256((__m256i *)(src + 3 * stride)); - dst[4] = _mm256_loadu_si256((__m256i *)(src + 4 * stride)); -} - -static inline void load_16bit_7rows_avx2(const int16_t *const src, - const ptrdiff_t stride, - __m256i dst[7]) { - dst[0] = _mm256_loadu_si256((__m256i *)(src + 0 * stride)); - dst[1] = _mm256_loadu_si256((__m256i *)(src + 1 * stride)); - dst[2] = _mm256_loadu_si256((__m256i *)(src + 2 * stride)); - dst[3] = _mm256_loadu_si256((__m256i *)(src + 3 * stride)); - dst[4] = _mm256_loadu_si256((__m256i *)(src + 4 * stride)); - dst[5] = _mm256_loadu_si256((__m256i *)(src + 5 * stride)); - dst[6] = _mm256_loadu_si256((__m256i *)(src + 6 * stride)); -} - -static AOM_FORCE_INLINE void load_16bit_8rows_avx2(const int16_t *const src, - const ptrdiff_t stride, - __m256i dst[8]) { - dst[0] = _mm256_loadu_si256((__m256i *)(src + 0 * stride)); - dst[1] = _mm256_loadu_si256((__m256i *)(src + 1 * stride)); - dst[2] = _mm256_loadu_si256((__m256i *)(src + 2 * stride)); - dst[3] = _mm256_loadu_si256((__m256i *)(src + 3 * stride)); - dst[4] = _mm256_loadu_si256((__m256i *)(src + 4 * stride)); - dst[5] = _mm256_loadu_si256((__m256i *)(src + 5 * stride)); - dst[6] = _mm256_loadu_si256((__m256i *)(src + 6 * stride)); - dst[7] = _mm256_loadu_si256((__m256i *)(src + 7 * stride)); -} - -static AOM_FORCE_INLINE void loadu_unpack_16bit_5rows_avx2( - const int16_t *const src, const ptrdiff_t stride, __m256i s_256[5], - __m256i ss_256[5], __m256i tt_256[5]) { - s_256[0] = _mm256_loadu_si256((__m256i *)(src + 0 * stride)); - s_256[1] = _mm256_loadu_si256((__m256i *)(src + 1 * stride)); - s_256[2] = _mm256_loadu_si256((__m256i *)(src + 2 * stride)); - s_256[3] = _mm256_loadu_si256((__m256i *)(src + 3 * stride)); - s_256[4] = _mm256_loadu_si256((__m256i *)(src + 4 * stride)); - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - ss_256[4] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - - tt_256[0] = _mm256_unpacklo_epi16(s_256[1], s_256[2]); - tt_256[1] = _mm256_unpacklo_epi16(s_256[3], s_256[4]); - tt_256[3] = _mm256_unpackhi_epi16(s_256[1], s_256[2]); - tt_256[4] = _mm256_unpackhi_epi16(s_256[3], s_256[4]); -} - -static AOM_FORCE_INLINE void loadu_unpack_16bit_3rows_avx2( - const int16_t *const src, const ptrdiff_t stride, __m256i s_256[3], - __m256i ss_256[3], __m256i tt_256[3]) { - s_256[0] = _mm256_loadu_si256((__m256i *)(src + 0 * stride)); - s_256[1] = _mm256_loadu_si256((__m256i *)(src + 1 * stride)); - s_256[2] = _mm256_loadu_si256((__m256i *)(src + 2 * stride)); - - ss_256[0] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[2] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - - tt_256[0] = _mm256_unpacklo_epi16(s_256[1], s_256[2]); - tt_256[2] = _mm256_unpackhi_epi16(s_256[1], s_256[2]); -} - -static inline void convolve_8tap_unpack_avx2(const __m256i s[6], - __m256i ss[7]) { - ss[0] = _mm256_unpacklo_epi16(s[0], s[1]); - ss[1] = _mm256_unpacklo_epi16(s[2], s[3]); - ss[2] = _mm256_unpacklo_epi16(s[4], s[5]); - ss[4] = _mm256_unpackhi_epi16(s[0], s[1]); - ss[5] = _mm256_unpackhi_epi16(s[2], s[3]); - ss[6] = _mm256_unpackhi_epi16(s[4], s[5]); -} - -static inline __m128i convolve_2tap_ssse3(const __m128i ss[1], - const __m128i coeffs[1]) { - return _mm_maddubs_epi16(ss[0], coeffs[0]); -} - -static inline __m128i convolve_4tap_ssse3(const __m128i ss[2], - const __m128i coeffs[2]) { - const __m128i res_23 = _mm_maddubs_epi16(ss[0], coeffs[0]); - const __m128i res_45 = _mm_maddubs_epi16(ss[1], coeffs[1]); - return _mm_add_epi16(res_23, res_45); -} - -static inline __m128i convolve_6tap_ssse3(const __m128i ss[3], - const __m128i coeffs[3]) { - const __m128i res_12 = _mm_maddubs_epi16(ss[0], coeffs[0]); - const __m128i res_34 = _mm_maddubs_epi16(ss[1], coeffs[1]); - const __m128i res_56 = _mm_maddubs_epi16(ss[2], coeffs[2]); - const __m128i res_1256 = _mm_add_epi16(res_12, res_56); - return _mm_add_epi16(res_1256, res_34); -} - -static inline __m128i convolve_8tap_ssse3(const __m128i ss[4], - const __m128i coeffs[4]) { - const __m128i res_01 = _mm_maddubs_epi16(ss[0], coeffs[0]); - const __m128i res_23 = _mm_maddubs_epi16(ss[1], coeffs[1]); - const __m128i res_45 = _mm_maddubs_epi16(ss[2], coeffs[2]); - const __m128i res_67 = _mm_maddubs_epi16(ss[3], coeffs[3]); - const __m128i res_0145 = _mm_add_epi16(res_01, res_45); - const __m128i res_2367 = _mm_add_epi16(res_23, res_67); - return _mm_add_epi16(res_0145, res_2367); -} - -static inline __m256i convolve_2tap_avx2(const __m256i ss[1], - const __m256i coeffs[1]) { - return _mm256_maddubs_epi16(ss[0], coeffs[0]); -} - -static inline __m256i convolve_4tap_avx2(const __m256i ss[2], - const __m256i coeffs[2]) { - const __m256i res_23 = _mm256_maddubs_epi16(ss[0], coeffs[0]); - const __m256i res_45 = _mm256_maddubs_epi16(ss[1], coeffs[1]); - return _mm256_add_epi16(res_23, res_45); -} - -static inline __m256i convolve_6tap_avx2(const __m256i ss[3], - const __m256i coeffs[3]) { - const __m256i res_01 = _mm256_maddubs_epi16(ss[0], coeffs[0]); - const __m256i res_23 = _mm256_maddubs_epi16(ss[1], coeffs[1]); - const __m256i res_45 = _mm256_maddubs_epi16(ss[2], coeffs[2]); - const __m256i res_0145 = _mm256_add_epi16(res_01, res_45); - return _mm256_add_epi16(res_0145, res_23); -} - -static inline __m256i convolve_8tap_avx2(const __m256i ss[4], - const __m256i coeffs[4]) { - const __m256i res_01 = _mm256_maddubs_epi16(ss[0], coeffs[0]); - const __m256i res_23 = _mm256_maddubs_epi16(ss[1], coeffs[1]); - const __m256i res_45 = _mm256_maddubs_epi16(ss[2], coeffs[2]); - const __m256i res_67 = _mm256_maddubs_epi16(ss[3], coeffs[3]); - const __m256i res_0145 = _mm256_add_epi16(res_01, res_45); - const __m256i res_2367 = _mm256_add_epi16(res_23, res_67); - return _mm256_add_epi16(res_0145, res_2367); -} - -static inline __m128i convolve16_2tap_sse2(const __m128i ss[1], - const __m128i coeffs[1]) { - return _mm_madd_epi16(ss[0], coeffs[0]); -} - -static inline __m128i convolve16_4tap_sse2(const __m128i ss[2], - const __m128i coeffs[2]) { - const __m128i res_01 = _mm_madd_epi16(ss[0], coeffs[0]); - const __m128i res_23 = _mm_madd_epi16(ss[1], coeffs[1]); - return _mm_add_epi32(res_01, res_23); -} - -static inline __m128i convolve16_6tap_sse2(const __m128i ss[3], - const __m128i coeffs[3]) { - const __m128i res_01 = _mm_madd_epi16(ss[0], coeffs[0]); - const __m128i res_23 = _mm_madd_epi16(ss[1], coeffs[1]); - const __m128i res_45 = _mm_madd_epi16(ss[2], coeffs[2]); - const __m128i res_0123 = _mm_add_epi32(res_01, res_23); - return _mm_add_epi32(res_0123, res_45); -} - -static inline __m128i convolve16_8tap_sse2(const __m128i ss[4], - const __m128i coeffs[4]) { - const __m128i res_01 = _mm_madd_epi16(ss[0], coeffs[0]); - const __m128i res_23 = _mm_madd_epi16(ss[1], coeffs[1]); - const __m128i res_45 = _mm_madd_epi16(ss[2], coeffs[2]); - const __m128i res_67 = _mm_madd_epi16(ss[3], coeffs[3]); - const __m128i res_0123 = _mm_add_epi32(res_01, res_23); - const __m128i res_4567 = _mm_add_epi32(res_45, res_67); - return _mm_add_epi32(res_0123, res_4567); -} - -static inline __m256i convolve16_2tap_avx2(const __m256i ss[1], - const __m256i coeffs[1]) { - return _mm256_madd_epi16(ss[0], coeffs[0]); -} - -static inline __m256i convolve16_4tap_avx2(const __m256i ss[2], - const __m256i coeffs[2]) { - const __m256i res_1 = _mm256_madd_epi16(ss[0], coeffs[0]); - const __m256i res_2 = _mm256_madd_epi16(ss[1], coeffs[1]); - return _mm256_add_epi32(res_1, res_2); -} - -static inline __m256i convolve16_6tap_avx2(const __m256i ss[3], - const __m256i coeffs[3]) { - const __m256i res_01 = _mm256_madd_epi16(ss[0], coeffs[0]); - const __m256i res_23 = _mm256_madd_epi16(ss[1], coeffs[1]); - const __m256i res_45 = _mm256_madd_epi16(ss[2], coeffs[2]); - const __m256i res_0123 = _mm256_add_epi32(res_01, res_23); - return _mm256_add_epi32(res_0123, res_45); -} - -static inline __m256i convolve16_8tap_avx2(const __m256i ss[4], - const __m256i coeffs[4]) { - const __m256i res_01 = _mm256_madd_epi16(ss[0], coeffs[0]); - const __m256i res_23 = _mm256_madd_epi16(ss[1], coeffs[1]); - const __m256i res_45 = _mm256_madd_epi16(ss[2], coeffs[2]); - const __m256i res_67 = _mm256_madd_epi16(ss[3], coeffs[3]); - const __m256i res_0123 = _mm256_add_epi32(res_01, res_23); - const __m256i res_4567 = _mm256_add_epi32(res_45, res_67); - return _mm256_add_epi32(res_0123, res_4567); -} - -static inline __m256i x_convolve_4tap_avx2(const __m256i data, - const __m256i coeffs[2], - const __m256i filt[2]) { - __m256i ss[2]; - - ss[0] = _mm256_shuffle_epi8(data, filt[0]); - ss[1] = _mm256_shuffle_epi8(data, filt[1]); - - return convolve_4tap_avx2(ss, coeffs); -} - -static inline __m256i x_convolve_6tap_avx2(const __m256i data, - const __m256i coeffs[3], - const __m256i filt[3]) { - __m256i ss[3]; - - ss[0] = _mm256_shuffle_epi8(data, filt[0]); - ss[1] = _mm256_shuffle_epi8(data, filt[1]); - ss[2] = _mm256_shuffle_epi8(data, filt[2]); - - return convolve_6tap_avx2(ss, coeffs); -} - -static inline __m256i x_convolve_8tap_avx2(const __m256i data, - const __m256i coeffs[4], - const __m256i filt[4]) { - __m256i ss[4]; - - ss[0] = _mm256_shuffle_epi8(data, filt[0]); - ss[1] = _mm256_shuffle_epi8(data, filt[1]); - ss[2] = _mm256_shuffle_epi8(data, filt[2]); - ss[3] = _mm256_shuffle_epi8(data, filt[3]); - - return convolve_8tap_avx2(ss, coeffs); -} - -static inline __m256i sr_y_round_avx2(const __m256i src) { - const __m256i round = _mm256_set1_epi16(32); - const __m256i dst = _mm256_add_epi16(src, round); - return _mm256_srai_epi16(dst, FILTER_BITS - 1); -} - -static inline __m128i xy_x_round_sse2(const __m128i src) { - const __m128i round = _mm_set1_epi16(2); - const __m128i dst = _mm_add_epi16(src, round); - return _mm_srai_epi16(dst, 2); -} - -static inline __m256i xy_x_round_avx2(const __m256i src) { - const __m256i round = _mm256_set1_epi16(2); - const __m256i dst = _mm256_add_epi16(src, round); - return _mm256_srai_epi16(dst, 2); -} - -static inline void xy_x_round_store_2x2_sse2(const __m128i res, - int16_t *const dst) { - const __m128i d = xy_x_round_sse2(res); - _mm_storel_epi64((__m128i *)dst, d); -} - -static inline void xy_x_round_store_4x2_sse2(const __m128i res, - int16_t *const dst) { - const __m128i d = xy_x_round_sse2(res); - _mm_storeu_si128((__m128i *)dst, d); -} - -static inline void xy_x_round_store_8x2_sse2(const __m128i res[2], - int16_t *const dst) { - __m128i r[2]; - - r[0] = xy_x_round_sse2(res[0]); - r[1] = xy_x_round_sse2(res[1]); - _mm_storeu_si128((__m128i *)dst, r[0]); - _mm_storeu_si128((__m128i *)(dst + 8), r[1]); -} - -static inline void xy_x_round_store_8x2_avx2(const __m256i res, - int16_t *const dst) { - const __m256i d = xy_x_round_avx2(res); - _mm256_storeu_si256((__m256i *)dst, d); -} - -static inline void xy_x_round_store_32_avx2(const __m256i res[2], - int16_t *const dst) { - __m256i r[2]; - - r[0] = xy_x_round_avx2(res[0]); - r[1] = xy_x_round_avx2(res[1]); - const __m256i d0 = - _mm256_inserti128_si256(r[0], _mm256_castsi256_si128(r[1]), 1); - const __m256i d1 = - _mm256_inserti128_si256(r[1], _mm256_extracti128_si256(r[0], 1), 0); - _mm256_storeu_si256((__m256i *)dst, d0); - _mm256_storeu_si256((__m256i *)(dst + 16), d1); -} - -static inline __m128i xy_y_round_sse2(const __m128i src) { - const __m128i round = _mm_set1_epi32(1024); - const __m128i dst = _mm_add_epi32(src, round); - return _mm_srai_epi32(dst, 11); -} - -static inline __m128i xy_y_round_half_pel_sse2(const __m128i src) { - const __m128i round = _mm_set1_epi16(16); - const __m128i dst = _mm_add_epi16(src, round); - return _mm_srai_epi16(dst, 5); -} - -static inline __m256i xy_y_round_avx2(const __m256i src) { - const __m256i round = _mm256_set1_epi32(1024); - const __m256i dst = _mm256_add_epi32(src, round); - return _mm256_srai_epi32(dst, 11); -} - -static inline __m256i xy_y_round_16_avx2(const __m256i r[2]) { - const __m256i r0 = xy_y_round_avx2(r[0]); - const __m256i r1 = xy_y_round_avx2(r[1]); - return _mm256_packs_epi32(r0, r1); -} - -static inline __m256i xy_y_round_half_pel_avx2(const __m256i src) { - const __m256i round = _mm256_set1_epi16(16); - const __m256i dst = _mm256_add_epi16(src, round); - return _mm256_srai_epi16(dst, 5); -} - -static inline void pack_store_2x2_sse2(const __m128i res, uint8_t *const dst, - const ptrdiff_t stride) { - const __m128i d = _mm_packus_epi16(res, res); - *(int16_t *)dst = (int16_t)_mm_cvtsi128_si32(d); - *(int16_t *)(dst + stride) = (int16_t)_mm_extract_epi16(d, 1); -} - -static inline void pack_store_4x2_sse2(const __m128i res, uint8_t *const dst, - const ptrdiff_t stride) { - const __m128i d = _mm_packus_epi16(res, res); - store_u8_4x2_sse2(d, dst, stride); -} - -static inline void pack_store_4x2_avx2(const __m256i res, uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i d = _mm256_packus_epi16(res, res); - const __m128i d0 = _mm256_castsi256_si128(d); - const __m128i d1 = _mm256_extracti128_si256(d, 1); - - xx_storel_32(dst, d0); - xx_storel_32(dst + stride, d1); -} - -static inline void pack_store_8x2_avx2(const __m256i res, uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i d = _mm256_packus_epi16(res, res); - const __m128i d0 = _mm256_castsi256_si128(d); - const __m128i d1 = _mm256_extracti128_si256(d, 1); - _mm_storel_epi64((__m128i *)dst, d0); - _mm_storel_epi64((__m128i *)(dst + stride), d1); -} - -static inline void pack_store_16x2_avx2(const __m256i res0, const __m256i res1, - uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i d = _mm256_packus_epi16(res0, res1); - storeu_u8_16x2_avx2(d, dst, stride); -} - -static inline void xy_y_pack_store_16x2_avx2(const __m256i res0, - const __m256i res1, - uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i t = _mm256_packus_epi16(res0, res1); - const __m256i d = _mm256_permute4x64_epi64(t, 0xD8); - storeu_u8_16x2_avx2(d, dst, stride); -} - -static inline void pack_store_32_avx2(const __m256i res0, const __m256i res1, - uint8_t *const dst) { - const __m256i t = _mm256_packus_epi16(res0, res1); - const __m256i d = _mm256_permute4x64_epi64(t, 0xD8); - _mm256_storeu_si256((__m256i *)dst, d); -} - -static inline void xy_y_round_store_2x2_sse2(const __m128i res, - uint8_t *const dst, - const ptrdiff_t stride) { - const __m128i r = xy_y_round_sse2(res); - const __m128i rr = _mm_packs_epi32(r, r); - pack_store_2x2_sse2(rr, dst, stride); -} - -static inline void xy_y_round_store_4x2_avx2(const __m256i res, - uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i r = xy_y_round_avx2(res); - const __m256i rr = _mm256_packs_epi32(r, r); - pack_store_4x2_avx2(rr, dst, stride); -} - -static inline void xy_y_pack_store_32_avx2(const __m256i res0, - const __m256i res1, - uint8_t *const dst) { - const __m256i d = _mm256_packus_epi16(res0, res1); - // d = _mm256_permute4x64_epi64(d, 0xD8); - _mm256_storeu_si256((__m256i *)dst, d); -} - -static inline void xy_y_round_store_32_avx2(const __m256i r0[2], - const __m256i r1[2], - uint8_t *const dst) { - const __m256i ra = xy_y_round_16_avx2(r0); - const __m256i rb = xy_y_round_16_avx2(r1); - xy_y_pack_store_32_avx2(ra, rb, dst); -} - -static inline void convolve_store_32_avx2(const __m256i res0, - const __m256i res1, - uint8_t *const dst) { - const __m256i d = _mm256_packus_epi16(res0, res1); - _mm256_storeu_si256((__m256i *)dst, d); -} - -static inline __m128i sr_x_round_sse2(const __m128i src) { - const __m128i round = _mm_set1_epi16(34); - const __m128i dst = _mm_add_epi16(src, round); - return _mm_srai_epi16(dst, 6); -} - -static inline __m256i sr_x_round_avx2(const __m256i src) { - const __m256i round = _mm256_set1_epi16(34); - const __m256i dst = _mm256_add_epi16(src, round); - return _mm256_srai_epi16(dst, 6); -} - -static inline __m128i sr_y_round_sse2(const __m128i src) { - const __m128i round = _mm_set1_epi16(32); - const __m128i dst = _mm_add_epi16(src, round); - return _mm_srai_epi16(dst, FILTER_BITS - 1); -} - -static inline void sr_x_round_store_8x2_avx2(const __m256i res, - uint8_t *const dst, - const ptrdiff_t dst_stride) { - const __m256i r = sr_x_round_avx2(res); - pack_store_8x2_avx2(r, dst, dst_stride); -} - -static inline void sr_x_round_store_16x2_avx2(const __m256i res[2], - uint8_t *const dst, - const ptrdiff_t dst_stride) { - __m256i r[2]; - - r[0] = sr_x_round_avx2(res[0]); - r[1] = sr_x_round_avx2(res[1]); - pack_store_16x2_avx2(r[0], r[1], dst, dst_stride); -} - -static inline void sr_x_round_store_32_avx2(const __m256i res[2], - uint8_t *const dst) { - __m256i r[2]; - - r[0] = sr_x_round_avx2(res[0]); - r[1] = sr_x_round_avx2(res[1]); - convolve_store_32_avx2(r[0], r[1], dst); -} - -static inline void sr_y_round_store_8x2_avx2(const __m256i res, - uint8_t *const dst, - const ptrdiff_t dst_stride) { - const __m256i r = sr_y_round_avx2(res); - pack_store_8x2_avx2(r, dst, dst_stride); -} - -static inline void sr_y_round_store_16x2_avx2(const __m256i res[2], - uint8_t *const dst, - const ptrdiff_t dst_stride) { - __m256i r[2]; - - r[0] = sr_y_round_avx2(res[0]); - r[1] = sr_y_round_avx2(res[1]); - pack_store_16x2_avx2(r[0], r[1], dst, dst_stride); -} - -static inline void sr_y_2tap_32_avg_avx2(const uint8_t *const src, - const __m256i s0, __m256i *const s1, - uint8_t *const dst) { - *s1 = _mm256_loadu_si256((__m256i *)src); - const __m256i d = _mm256_avg_epu8(s0, *s1); - _mm256_storeu_si256((__m256i *)dst, d); -} - -static inline void sr_x_2tap_32_avg_avx2(const uint8_t *const src, - uint8_t *const dst) { - const __m256i s0 = _mm256_loadu_si256((__m256i *)src); - const __m256i s1 = _mm256_loadu_si256((__m256i *)(src + 1)); - const __m256i d = _mm256_avg_epu8(s0, s1); - _mm256_storeu_si256((__m256i *)dst, d); -} - -static inline __m128i x_convolve_2tap_2x2_sse4_1(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[1]) { - const __m128i sfl = - _mm_setr_epi8(0, 1, 1, 2, 4, 5, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i s_128 = load_u8_4x2_sse4_1(src, stride); - const __m128i ss = _mm_shuffle_epi8(s_128, sfl); - return convolve_2tap_ssse3(&ss, coeffs); -} - -static inline __m128i x_convolve_2tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[1]) { - const __m128i sfl = - _mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12); - const __m128i s_128 = load_u8_8x2_sse2(src, stride); - const __m128i ss = _mm_shuffle_epi8(s_128, sfl); - return convolve_2tap_ssse3(&ss, coeffs); -} - -static inline void x_convolve_2tap_8x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[1], - __m128i r[2]) { - __m128i ss[2]; - const __m128i s00 = _mm_loadu_si128((__m128i *)src); - const __m128i s10 = _mm_loadu_si128((__m128i *)(src + stride)); - const __m128i s01 = _mm_srli_si128(s00, 1); - const __m128i s11 = _mm_srli_si128(s10, 1); - ss[0] = _mm_unpacklo_epi8(s00, s01); - ss[1] = _mm_unpacklo_epi8(s10, s11); - - r[0] = convolve_2tap_ssse3(&ss[0], coeffs); - r[1] = convolve_2tap_ssse3(&ss[1], coeffs); -} - -static inline __m256i x_convolve_2tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[1]) { - __m128i s_128[2][2]; - __m256i s_256[2]; - - s_128[0][0] = _mm_loadu_si128((__m128i *)src); - s_128[1][0] = _mm_loadu_si128((__m128i *)(src + stride)); - s_128[0][1] = _mm_srli_si128(s_128[0][0], 1); - s_128[1][1] = _mm_srli_si128(s_128[1][0], 1); - s_256[0] = _mm256_setr_m128i(s_128[0][0], s_128[1][0]); - s_256[1] = _mm256_setr_m128i(s_128[0][1], s_128[1][1]); - const __m256i ss = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - return convolve_2tap_avx2(&ss, coeffs); -} - -static inline void x_convolve_2tap_16x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[1], - __m256i r[2]) { - const __m256i s0_256 = loadu_8bit_16x2_avx2(src, stride); - const __m256i s1_256 = loadu_8bit_16x2_avx2(src + 1, stride); - const __m256i s0 = _mm256_unpacklo_epi8(s0_256, s1_256); - const __m256i s1 = _mm256_unpackhi_epi8(s0_256, s1_256); - r[0] = convolve_2tap_avx2(&s0, coeffs); - r[1] = convolve_2tap_avx2(&s1, coeffs); -} - -static inline void x_convolve_2tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[1], - __m256i r[2]) { - const __m256i s0 = _mm256_loadu_si256((__m256i *)src); - const __m256i s1 = _mm256_loadu_si256((__m256i *)(src + 1)); - const __m256i ss0 = _mm256_unpacklo_epi8(s0, s1); - const __m256i ss1 = _mm256_unpackhi_epi8(s0, s1); - - r[0] = convolve_2tap_avx2(&ss0, coeffs); - r[1] = convolve_2tap_avx2(&ss1, coeffs); -} - -static inline __m128i x_convolve_4tap_2x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[2]) { - const __m128i sfl0 = - _mm_setr_epi8(0, 1, 1, 2, 8, 9, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i sfl1 = - _mm_setr_epi8(2, 3, 3, 4, 10, 11, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i s = load_u8_8x2_sse2(src, stride); - __m128i ss[2]; - - ss[0] = _mm_shuffle_epi8(s, sfl0); - ss[1] = _mm_shuffle_epi8(s, sfl1); - return convolve_4tap_ssse3(ss, coeffs); -} - -static inline __m128i x_convolve_4tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[2]) { - const __m128i s = load_u8_8x2_sse2(src, stride); - const __m128i sfl0 = - _mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12); - const __m128i sfl1 = - _mm_setr_epi8(2, 3, 3, 4, 4, 5, 5, 6, 10, 11, 11, 12, 12, 13, 13, 14); - __m128i ss[2]; - - ss[0] = _mm_shuffle_epi8(s, sfl0); - ss[1] = _mm_shuffle_epi8(s, sfl1); - return convolve_4tap_ssse3(ss, coeffs); -} - -static inline __m256i x_convolve_4tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[2], - const __m256i filt[2]) { - const __m256i s_256 = loadu_8bit_16x2_avx2(src, stride); - return x_convolve_4tap_avx2(s_256, coeffs, filt); -} - -static inline void x_convolve_4tap_16x2_avx2(const uint8_t *const src, - const int32_t src_stride, - const __m256i coeffs[2], - const __m256i filt[2], - __m256i r[2]) { - r[0] = x_convolve_4tap_8x2_avx2(src + 0, src_stride, coeffs, filt); - r[1] = x_convolve_4tap_8x2_avx2(src + 8, src_stride, coeffs, filt); -} - -static inline void x_convolve_4tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[2], - const __m256i filt[2], - __m256i r[2]) { - const __m256i s0_256 = _mm256_loadu_si256((__m256i *)src); - const __m256i s1_256 = _mm256_loadu_si256((__m256i *)(src + 8)); - - r[0] = x_convolve_4tap_avx2(s0_256, coeffs, filt); - r[1] = x_convolve_4tap_avx2(s1_256, coeffs, filt); -} - -static inline __m128i x_convolve_6tap_2x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[3]) { - const __m128i sfl0 = - _mm_setr_epi8(0, 1, 1, 2, 8, 9, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i sfl1 = - _mm_setr_epi8(2, 3, 3, 4, 10, 11, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i sfl2 = - _mm_setr_epi8(4, 5, 5, 6, 12, 13, 13, 14, 0, 0, 0, 0, 0, 0, 0, 0); - - const __m128i s = load_u8_8x2_sse2(src, stride); - __m128i ss[3]; - - ss[0] = _mm_shuffle_epi8(s, sfl0); - ss[1] = _mm_shuffle_epi8(s, sfl1); - ss[2] = _mm_shuffle_epi8(s, sfl2); - return convolve_6tap_ssse3(ss, coeffs); -} - -static inline __m128i x_convolve_6tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[3]) { - const __m128i s = load_u8_8x2_sse2(src, stride); - const __m128i sfl0 = - _mm_setr_epi8(0, 1, 1, 2, 8, 9, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i sfl1 = - _mm_setr_epi8(2, 3, 3, 4, 10, 11, 11, 12, 0, 0, 0, 0, 0, 0, 0, 0); - const __m128i sfl2 = - _mm_setr_epi8(4, 5, 5, 6, 12, 13, 13, 14, 0, 0, 0, 0, 0, 0, 0, 0); - __m128i ss[3]; - - ss[0] = _mm_shuffle_epi8(s, sfl0); - ss[1] = _mm_shuffle_epi8(s, sfl1); - ss[2] = _mm_shuffle_epi8(s, sfl2); - return convolve_6tap_ssse3(ss, coeffs); -} - -static inline __m256i x_convolve_6tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[3], - const __m256i filt[3]) { - const __m256i s_256 = loadu_8bit_16x2_avx2(src, stride); - return x_convolve_6tap_avx2(s_256, coeffs, filt); -} - -static inline void x_convolve_6tap_16x2_avx2(const uint8_t *const src, - const int32_t src_stride, - const __m256i coeffs[3], - const __m256i filt[3], - __m256i r[2]) { - r[0] = x_convolve_6tap_8x2_avx2(src + 0, src_stride, coeffs, filt); - r[1] = x_convolve_6tap_8x2_avx2(src + 8, src_stride, coeffs, filt); -} - -static inline void x_convolve_6tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[3], - const __m256i filt[3], - __m256i r[2]) { - const __m256i s0_256 = _mm256_loadu_si256((__m256i *)src); - const __m256i s1_256 = _mm256_loadu_si256((__m256i *)(src + 8)); - - r[0] = x_convolve_6tap_avx2(s0_256, coeffs, filt); - r[1] = x_convolve_6tap_avx2(s1_256, coeffs, filt); -} - -static inline __m256i x_convolve_8tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[4], - const __m256i filt[4]) { - const __m256i s_256 = loadu_8bit_16x2_avx2(src, stride); - return x_convolve_8tap_avx2(s_256, coeffs, filt); -} - -static AOM_FORCE_INLINE void x_convolve_8tap_16x2_avx2(const uint8_t *const src, - const int32_t src_stride, - const __m256i coeffs[4], - const __m256i filt[4], - __m256i r[2]) { - r[0] = x_convolve_8tap_8x2_avx2(src + 0, src_stride, coeffs, filt); - r[1] = x_convolve_8tap_8x2_avx2(src + 8, src_stride, coeffs, filt); -} - -static AOM_FORCE_INLINE void x_convolve_8tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[4], - const __m256i filt[4], - __m256i r[2]) { - const __m256i s0_256 = _mm256_loadu_si256((__m256i *)src); - const __m256i s1_256 = _mm256_loadu_si256((__m256i *)(src + 8)); - - r[0] = x_convolve_8tap_avx2(s0_256, coeffs, filt); - r[1] = x_convolve_8tap_avx2(s1_256, coeffs, filt); -} - -static inline __m128i y_convolve_2tap_2x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[1], - __m128i s_16[2]) { - __m128i s_128[2]; - - s_16[1] = _mm_cvtsi32_si128(*(int16_t *)(src + stride)); - s_128[0] = _mm_unpacklo_epi16(s_16[0], s_16[1]); - s_16[0] = _mm_cvtsi32_si128(*(int16_t *)(src + 2 * stride)); - s_128[1] = _mm_unpacklo_epi16(s_16[1], s_16[0]); - const __m128i ss = _mm_unpacklo_epi8(s_128[0], s_128[1]); - return convolve_2tap_ssse3(&ss, coeffs); -} - -static inline __m128i y_convolve_2tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[1], - __m128i s_32[2]) { - __m128i s_128[2]; - - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src + stride)); - s_128[0] = _mm_unpacklo_epi32(s_32[0], s_32[1]); - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src + 2 * stride)); - s_128[1] = _mm_unpacklo_epi32(s_32[1], s_32[0]); - const __m128i ss = _mm_unpacklo_epi8(s_128[0], s_128[1]); - return convolve_2tap_ssse3(&ss, coeffs); -} - -static inline __m256i y_convolve_2tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[1], - __m128i s_64[2]) { - __m256i s_256[2]; - - s_64[1] = _mm_loadl_epi64((__m128i *)(src + stride)); - s_256[0] = _mm256_setr_m128i(s_64[0], s_64[1]); - s_64[0] = _mm_loadl_epi64((__m128i *)(src + 2 * stride)); - s_256[1] = _mm256_setr_m128i(s_64[1], s_64[0]); - const __m256i ss = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - return convolve_2tap_avx2(&ss, coeffs); -} - -static inline void y_convolve_2tap_16x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[1], - __m128i s_128[2], __m256i r[2]) { - __m256i s_256[2]; - - s_128[1] = _mm_loadu_si128((__m128i *)(src + stride)); - s_256[0] = _mm256_setr_m128i(s_128[0], s_128[1]); - s_128[0] = _mm_loadu_si128((__m128i *)(src + 2 * stride)); - s_256[1] = _mm256_setr_m128i(s_128[1], s_128[0]); - const __m256i ss0 = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - const __m256i ss1 = _mm256_unpackhi_epi8(s_256[0], s_256[1]); - r[0] = convolve_2tap_avx2(&ss0, coeffs); - r[1] = convolve_2tap_avx2(&ss1, coeffs); -} - -static inline void y_convolve_2tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[1], - const __m256i s0, __m256i *const s1, - __m256i r[2]) { - *s1 = _mm256_loadu_si256((__m256i *)src); - const __m256i ss0 = _mm256_unpacklo_epi8(s0, *s1); - const __m256i ss1 = _mm256_unpackhi_epi8(s0, *s1); - r[0] = convolve_2tap_avx2(&ss0, coeffs); - r[1] = convolve_2tap_avx2(&ss1, coeffs); -} - -static inline __m128i y_convolve_4tap_2x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[2], - __m128i s_16[4], - __m128i ss_128[2]) { - s_16[3] = _mm_cvtsi32_si128(loadu_int16(src + stride)); - const __m128i src23 = _mm_unpacklo_epi16(s_16[2], s_16[3]); - s_16[2] = _mm_cvtsi32_si128(loadu_int16(src + 2 * stride)); - const __m128i src34 = _mm_unpacklo_epi16(s_16[3], s_16[2]); - ss_128[1] = _mm_unpacklo_epi8(src23, src34); - return convolve_4tap_ssse3(ss_128, coeffs); -} - -static inline __m128i y_convolve_4tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[2], - __m128i s_32[4], - __m128i ss_128[2]) { - s_32[3] = _mm_cvtsi32_si128(loadu_int32(src + stride)); - const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]); - s_32[2] = _mm_cvtsi32_si128(loadu_int32(src + 2 * stride)); - const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[2]); - ss_128[1] = _mm_unpacklo_epi8(src23, src34); - return convolve_4tap_ssse3(ss_128, coeffs); -} - -static inline __m256i y_convolve_4tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[2], - __m128i s_64[4], - __m256i ss_256[2]) { - s_64[3] = _mm_loadl_epi64((__m128i *)(src + stride)); - const __m256i src23 = _mm256_setr_m128i(s_64[2], s_64[3]); - s_64[2] = _mm_loadl_epi64((__m128i *)(src + 2 * stride)); - const __m256i src34 = _mm256_setr_m128i(s_64[3], s_64[2]); - ss_256[1] = _mm256_unpacklo_epi8(src23, src34); - return convolve_4tap_avx2(ss_256, coeffs); -} - -static inline void y_convolve_4tap_16x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[2], - __m128i s_128[4], - __m256i ss_256[4], __m256i r[2]) { - s_128[3] = _mm_loadu_si128((__m128i *)(src + stride)); - const __m256i src23 = _mm256_setr_m128i(s_128[2], s_128[3]); - s_128[2] = _mm_loadu_si128((__m128i *)(src + 2 * stride)); - const __m256i src34 = _mm256_setr_m128i(s_128[3], s_128[2]); - ss_256[1] = _mm256_unpacklo_epi8(src23, src34); - ss_256[3] = _mm256_unpackhi_epi8(src23, src34); - r[0] = convolve_4tap_avx2(ss_256, coeffs); - r[1] = convolve_4tap_avx2(ss_256 + 2, coeffs); -} - -static inline __m128i y_convolve_6tap_2x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[3], - __m128i s_16[6], - __m128i ss_128[3]) { - s_16[5] = _mm_cvtsi32_si128(loadu_int16(src + 3 * stride)); - const __m128i src45 = _mm_unpacklo_epi16(s_16[4], s_16[5]); - s_16[4] = _mm_cvtsi32_si128(loadu_int16(src + 4 * stride)); - const __m128i src56 = _mm_unpacklo_epi16(s_16[5], s_16[4]); - ss_128[2] = _mm_unpacklo_epi8(src45, src56); - return convolve_6tap_ssse3(ss_128, coeffs); -} - -static inline void y_convolve_4tap_32x2_avx2( - const uint8_t *const src, const ptrdiff_t stride, const __m256i coeffs[2], - __m256i s_256[4], __m256i ss_256[4], __m256i tt_256[4], __m256i r[4]) { - s_256[3] = _mm256_loadu_si256((__m256i *)(src + 1 * stride)); - ss_256[1] = _mm256_unpacklo_epi8(s_256[2], s_256[3]); - ss_256[3] = _mm256_unpackhi_epi8(s_256[2], s_256[3]); - s_256[2] = _mm256_loadu_si256((__m256i *)(src + 2 * stride)); - tt_256[1] = _mm256_unpacklo_epi8(s_256[3], s_256[2]); - tt_256[3] = _mm256_unpackhi_epi8(s_256[3], s_256[2]); - r[0] = convolve_4tap_avx2(ss_256 + 0, coeffs); - r[1] = convolve_4tap_avx2(ss_256 + 2, coeffs); - r[2] = convolve_4tap_avx2(tt_256 + 0, coeffs); - r[3] = convolve_4tap_avx2(tt_256 + 2, coeffs); -} - -static inline __m128i y_convolve_6tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[3], - __m128i s_32[6], - __m128i ss_128[3]) { - s_32[5] = _mm_cvtsi32_si128(loadu_int32(src + 3 * stride)); - const __m128i src45 = _mm_unpacklo_epi32(s_32[4], s_32[5]); - s_32[4] = _mm_cvtsi32_si128(loadu_int32(src + 4 * stride)); - const __m128i src56 = _mm_unpacklo_epi32(s_32[5], s_32[4]); - ss_128[2] = _mm_unpacklo_epi8(src45, src56); - return convolve_6tap_ssse3(ss_128, coeffs); -} - -static inline __m256i y_convolve_6tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[3], - __m128i s_64[6], - __m256i ss_256[3]) { - s_64[5] = _mm_loadl_epi64((__m128i *)(src + 3 * stride)); - const __m256i src45 = _mm256_setr_m128i(s_64[4], s_64[5]); - s_64[4] = _mm_loadl_epi64((__m128i *)(src + 4 * stride)); - const __m256i src56 = _mm256_setr_m128i(s_64[5], s_64[4]); - ss_256[2] = _mm256_unpacklo_epi8(src45, src56); - return convolve_6tap_avx2(ss_256, coeffs); -} - -static inline void y_convolve_6tap_16x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[3], - __m128i s_128[6], - __m256i ss_256[6], __m256i r[2]) { - s_128[5] = _mm_loadu_si128((__m128i *)(src + 3 * stride)); - const __m256i src45 = _mm256_setr_m128i(s_128[4], s_128[5]); - s_128[4] = _mm_loadu_si128((__m128i *)(src + 4 * stride)); - const __m256i src56 = _mm256_setr_m128i(s_128[5], s_128[4]); - ss_256[2] = _mm256_unpacklo_epi8(src45, src56); - ss_256[5] = _mm256_unpackhi_epi8(src45, src56); - r[0] = convolve_6tap_avx2(ss_256, coeffs); - r[1] = convolve_6tap_avx2(ss_256 + 3, coeffs); -} - -static inline void y_convolve_6tap_32x2_avx2( - const uint8_t *const src, const ptrdiff_t stride, const __m256i coeffs[3], - __m256i s_256[6], __m256i ss_256[6], __m256i tt_256[6], __m256i r[4]) { - s_256[5] = _mm256_loadu_si256((__m256i *)(src + 3 * stride)); - ss_256[2] = _mm256_unpacklo_epi8(s_256[4], s_256[5]); - ss_256[5] = _mm256_unpackhi_epi8(s_256[4], s_256[5]); - s_256[4] = _mm256_loadu_si256((__m256i *)(src + 4 * stride)); - tt_256[2] = _mm256_unpacklo_epi8(s_256[5], s_256[4]); - tt_256[5] = _mm256_unpackhi_epi8(s_256[5], s_256[4]); - r[0] = convolve_6tap_avx2(ss_256 + 0, coeffs); - r[1] = convolve_6tap_avx2(ss_256 + 3, coeffs); - r[2] = convolve_6tap_avx2(tt_256 + 0, coeffs); - r[3] = convolve_6tap_avx2(tt_256 + 3, coeffs); -} - -static inline __m128i y_convolve_8tap_2x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[4], - __m128i s_16[8], - __m128i ss_128[4]) { - s_16[7] = _mm_cvtsi32_si128(loadu_int16(src + 7 * stride)); - const __m128i src67 = _mm_unpacklo_epi16(s_16[6], s_16[7]); - s_16[6] = _mm_cvtsi32_si128(loadu_int16(src + 8 * stride)); - const __m128i src78 = _mm_unpacklo_epi16(s_16[7], s_16[6]); - ss_128[3] = _mm_unpacklo_epi8(src67, src78); - return convolve_8tap_ssse3(ss_128, coeffs); -} - -static inline __m128i y_convolve_8tap_4x2_ssse3(const uint8_t *const src, - const ptrdiff_t stride, - const __m128i coeffs[4], - __m128i s_32[8], - __m128i ss_128[4]) { - s_32[7] = _mm_cvtsi32_si128(loadu_int32(src + 7 * stride)); - const __m128i src67 = _mm_unpacklo_epi32(s_32[6], s_32[7]); - s_32[6] = _mm_cvtsi32_si128(loadu_int32(src + 8 * stride)); - const __m128i src78 = _mm_unpacklo_epi32(s_32[7], s_32[6]); - ss_128[3] = _mm_unpacklo_epi8(src67, src78); - return convolve_8tap_ssse3(ss_128, coeffs); -} - -static inline __m256i y_convolve_8tap_8x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[4], - __m128i s_64[8], - __m256i ss_256[4]) { - s_64[7] = _mm_loadl_epi64((__m128i *)(src + 7 * stride)); - const __m256i src67 = _mm256_setr_m128i(s_64[6], s_64[7]); - s_64[6] = _mm_loadl_epi64((__m128i *)(src + 8 * stride)); - const __m256i src78 = _mm256_setr_m128i(s_64[7], s_64[6]); - ss_256[3] = _mm256_unpacklo_epi8(src67, src78); - return convolve_8tap_avx2(ss_256, coeffs); -} - -static inline void y_convolve_8tap_16x2_avx2(const uint8_t *const src, - const ptrdiff_t stride, - const __m256i coeffs[4], - __m128i s_128[8], - __m256i ss_256[8], __m256i r[2]) { - s_128[7] = _mm_loadu_si128((__m128i *)(src + 7 * stride)); - const __m256i src67 = _mm256_setr_m128i(s_128[6], s_128[7]); - s_128[6] = _mm_loadu_si128((__m128i *)(src + 8 * stride)); - const __m256i src78 = _mm256_setr_m128i(s_128[7], s_128[6]); - ss_256[3] = _mm256_unpacklo_epi8(src67, src78); - ss_256[7] = _mm256_unpackhi_epi8(src67, src78); - r[0] = convolve_8tap_avx2(ss_256, coeffs); - r[1] = convolve_8tap_avx2(ss_256 + 4, coeffs); -} - -static inline void y_convolve_8tap_32x2_avx2( - const uint8_t *const src, const ptrdiff_t stride, const __m256i coeffs[4], - __m256i s_256[8], __m256i ss_256[8], __m256i tt_256[8], __m256i r[4]) { - s_256[7] = _mm256_loadu_si256((__m256i *)(src + 7 * stride)); - ss_256[3] = _mm256_unpacklo_epi8(s_256[6], s_256[7]); - ss_256[7] = _mm256_unpackhi_epi8(s_256[6], s_256[7]); - s_256[6] = _mm256_loadu_si256((__m256i *)(src + 8 * stride)); - tt_256[3] = _mm256_unpacklo_epi8(s_256[7], s_256[6]); - tt_256[7] = _mm256_unpackhi_epi8(s_256[7], s_256[6]); - r[0] = convolve_8tap_avx2(ss_256 + 0, coeffs); - r[1] = convolve_8tap_avx2(ss_256 + 4, coeffs); - r[2] = convolve_8tap_avx2(tt_256 + 0, coeffs); - r[3] = convolve_8tap_avx2(tt_256 + 4, coeffs); -} - -static inline void xy_x_convolve_2tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[1], - __m256i r[2]) { - const __m256i s0 = _mm256_loadu_si256((__m256i *)src); - const __m256i s1 = _mm256_loadu_si256((__m256i *)(src + 1)); - const __m256i ss0 = _mm256_unpacklo_epi8(s0, s1); - const __m256i ss1 = _mm256_unpackhi_epi8(s0, s1); - - r[0] = convolve_2tap_avx2(&ss0, coeffs); - r[1] = convolve_2tap_avx2(&ss1, coeffs); -} - -static inline void xy_x_2tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[1], - int16_t *const dst) { - __m256i r[2]; - - xy_x_convolve_2tap_32_avx2(src, coeffs, r); - const __m256i d0 = xy_x_round_avx2(r[0]); - const __m256i d1 = xy_x_round_avx2(r[1]); - _mm256_storeu_si256((__m256i *)dst, d0); - _mm256_storeu_si256((__m256i *)(dst + 16), d1); -} - -static inline void xy_x_4tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[2], - const __m256i filt[2], - int16_t *const dst) { - __m256i r[2]; - - x_convolve_4tap_32_avx2(src, coeffs, filt, r); - const __m256i d0 = xy_x_round_avx2(r[0]); - const __m256i d1 = xy_x_round_avx2(r[1]); - _mm256_storeu_si256((__m256i *)dst, d0); - _mm256_storeu_si256((__m256i *)(dst + 16), d1); -} - -static inline void xy_x_6tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[3], - const __m256i filt[3], - int16_t *const dst) { - __m256i r[2]; - - x_convolve_6tap_32_avx2(src, coeffs, filt, r); - const __m256i d0 = xy_x_round_avx2(r[0]); - const __m256i d1 = xy_x_round_avx2(r[1]); - _mm256_storeu_si256((__m256i *)dst, d0); - _mm256_storeu_si256((__m256i *)(dst + 16), d1); -} - -static inline void xy_x_8tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[4], - const __m256i filt[4], - int16_t *const dst) { - __m256i r[2]; - - x_convolve_8tap_32_avx2(src, coeffs, filt, r); - const __m256i d0 = xy_x_round_avx2(r[0]); - const __m256i d1 = xy_x_round_avx2(r[1]); - _mm256_storeu_si256((__m256i *)dst, d0); - _mm256_storeu_si256((__m256i *)(dst + 16), d1); -} - -static inline __m128i xy_y_convolve_2tap_2x2_sse2(const int16_t *const src, - __m128i s_32[2], - const __m128i coeffs[1]) { - __m128i s_128[2]; - - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src + 2)); - s_128[0] = _mm_unpacklo_epi32(s_32[0], s_32[1]); - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src + 2 * 2)); - s_128[1] = _mm_unpacklo_epi32(s_32[1], s_32[0]); - const __m128i ss = _mm_unpacklo_epi16(s_128[0], s_128[1]); - return convolve16_2tap_sse2(&ss, coeffs); -} - -static inline __m128i xy_y_convolve_2tap_2x2_half_pel_sse2( - const int16_t *const src, __m128i s_32[2]) { - __m128i s_128[2]; - - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src + 2)); - s_128[0] = _mm_unpacklo_epi32(s_32[0], s_32[1]); - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src + 2 * 2)); - s_128[1] = _mm_unpacklo_epi32(s_32[1], s_32[0]); - return _mm_add_epi16(s_128[0], s_128[1]); -} - -static inline void xy_y_convolve_2tap_4x2_sse2(const int16_t *const src, - __m128i s_64[2], - const __m128i coeffs[1], - __m128i r[2]) { - __m128i s_128[2]; - - s_64[1] = _mm_loadl_epi64((__m128i *)(src + 4)); - s_128[0] = _mm_unpacklo_epi64(s_64[0], s_64[1]); - s_64[0] = _mm_loadl_epi64((__m128i *)(src + 2 * 4)); - s_128[1] = _mm_unpacklo_epi64(s_64[1], s_64[0]); - const __m128i ss0 = _mm_unpacklo_epi16(s_128[0], s_128[1]); - const __m128i ss1 = _mm_unpackhi_epi16(s_128[0], s_128[1]); - r[0] = convolve16_2tap_sse2(&ss0, coeffs); - r[1] = convolve16_2tap_sse2(&ss1, coeffs); -} - -static inline __m128i xy_y_convolve_2tap_4x2_half_pel_sse2( - const int16_t *const src, __m128i s_64[2]) { - __m128i s_128[2]; - - s_64[1] = _mm_loadl_epi64((__m128i *)(src + 4)); - s_128[0] = _mm_unpacklo_epi64(s_64[0], s_64[1]); - s_64[0] = _mm_loadl_epi64((__m128i *)(src + 2 * 4)); - s_128[1] = _mm_unpacklo_epi64(s_64[1], s_64[0]); - return _mm_add_epi16(s_128[0], s_128[1]); -} - -static inline void xy_y_convolve_2tap_16_avx2(const __m256i s0, - const __m256i s1, - const __m256i coeffs[1], - __m256i r[2]) { - const __m256i ss0 = _mm256_unpacklo_epi16(s0, s1); - const __m256i ss1 = _mm256_unpackhi_epi16(s0, s1); - r[0] = convolve16_2tap_avx2(&ss0, coeffs); - r[1] = convolve16_2tap_avx2(&ss1, coeffs); -} - -static inline void xy_y_convolve_2tap_8x2_avx2(const int16_t *const src, - __m128i s_128[2], - const __m256i coeffs[1], - __m256i r[2]) { - __m256i s_256[2]; - s_128[1] = _mm_loadu_si128((__m128i *)(src + 8)); - s_256[0] = _mm256_setr_m128i(s_128[0], s_128[1]); - s_128[0] = _mm_loadu_si128((__m128i *)(src + 2 * 8)); - s_256[1] = _mm256_setr_m128i(s_128[1], s_128[0]); - xy_y_convolve_2tap_16_avx2(s_256[0], s_256[1], coeffs, r); -} - -static inline __m256i xy_y_convolve_2tap_8x2_half_pel_avx2( - const int16_t *const src, __m128i s_128[2]) { - __m256i s_256[2]; - s_128[1] = _mm_loadu_si128((__m128i *)(src + 8)); - s_256[0] = _mm256_setr_m128i(s_128[0], s_128[1]); - s_128[0] = _mm_loadu_si128((__m128i *)(src + 2 * 8)); - s_256[1] = _mm256_setr_m128i(s_128[1], s_128[0]); - return _mm256_add_epi16(s_256[0], s_256[1]); -} - -static inline void xy_y_convolve_2tap_16x2_half_pel_avx2( - const int16_t *const src, __m256i s_256[2], __m256i r[2]) { - s_256[1] = _mm256_loadu_si256((__m256i *)(src + 16)); - r[0] = _mm256_add_epi16(s_256[0], s_256[1]); - s_256[0] = _mm256_loadu_si256((__m256i *)(src + 2 * 16)); - r[1] = _mm256_add_epi16(s_256[1], s_256[0]); -} - -static inline void xy_y_store_16x2_avx2(const __m256i r[2], uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i t = _mm256_packus_epi16(r[0], r[1]); - const __m256i d = _mm256_permute4x64_epi64(t, 0xD8); - storeu_u8_16x2_avx2(d, dst, stride); -} - -static inline void xy_y_convolve_2tap_16x2_avx2(const int16_t *const src, - __m256i s[2], - const __m256i coeffs[1], - __m256i r[4]) { - s[1] = _mm256_loadu_si256((__m256i *)(src + 16)); - xy_y_convolve_2tap_16_avx2(s[0], s[1], coeffs, r + 0); - s[0] = _mm256_loadu_si256((__m256i *)(src + 2 * 16)); - xy_y_convolve_2tap_16_avx2(s[1], s[0], coeffs, r + 2); -} - -static inline void xy_y_convolve_2tap_32_avx2(const int16_t *const src, - const __m256i s0[2], - __m256i s1[2], - const __m256i coeffs[1], - __m256i r[4]) { - s1[0] = _mm256_loadu_si256((__m256i *)src); - s1[1] = _mm256_loadu_si256((__m256i *)(src + 16)); - xy_y_convolve_2tap_16_avx2(s0[0], s1[0], coeffs, r + 0); - xy_y_convolve_2tap_16_avx2(s0[1], s1[1], coeffs, r + 2); -} - -static inline void xy_y_convolve_2tap_32_all_avx2(const int16_t *const src, - const __m256i s0[2], - __m256i s1[2], - const __m256i coeffs[1], - uint8_t *const dst) { - __m256i r[4]; - - xy_y_convolve_2tap_32_avx2(src, s0, s1, coeffs, r); - xy_y_round_store_32_avx2(r + 0, r + 2, dst); -} - -static inline void xy_y_convolve_2tap_half_pel_32_avx2(const int16_t *const src, - const __m256i s0[2], - __m256i s1[2], - __m256i r[2]) { - s1[0] = _mm256_loadu_si256((__m256i *)src); - s1[1] = _mm256_loadu_si256((__m256i *)(src + 16)); - r[0] = _mm256_add_epi16(s0[0], s1[0]); - r[1] = _mm256_add_epi16(s0[1], s1[1]); -} - -static inline void xy_y_convolve_2tap_half_pel_32_all_avx2( - const int16_t *const src, const __m256i s0[2], __m256i s1[2], - uint8_t *const dst) { - __m256i r[2]; - - xy_y_convolve_2tap_half_pel_32_avx2(src, s0, s1, r); - r[0] = xy_y_round_half_pel_avx2(r[0]); - r[1] = xy_y_round_half_pel_avx2(r[1]); - xy_y_pack_store_32_avx2(r[0], r[1], dst); -} - -static inline __m128i xy_y_convolve_4tap_2x2_sse2(const int16_t *const src, - __m128i s_32[4], - __m128i ss_128[2], - const __m128i coeffs[2]) { - s_32[3] = _mm_cvtsi32_si128(loadu_int32(src + 3 * 2)); - const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]); - s_32[2] = _mm_cvtsi32_si128(loadu_int32(src + 4 * 2)); - const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[2]); - ss_128[1] = _mm_unpacklo_epi16(src23, src34); - const __m128i r = convolve16_4tap_sse2(ss_128, coeffs); - ss_128[0] = ss_128[1]; - return r; -} - -static inline __m256i xy_y_convolve_4tap_4x2_avx2(const int16_t *const src, - __m128i s_64[4], - __m256i ss_256[2], - const __m256i coeffs[2]) { - __m256i s_256[2]; - s_64[3] = _mm_loadl_epi64((__m128i *)(src + 3 * 4)); - s_256[0] = _mm256_setr_m128i(s_64[2], s_64[3]); - s_64[2] = _mm_loadl_epi64((__m128i *)(src + 4 * 4)); - s_256[1] = _mm256_setr_m128i(s_64[3], s_64[2]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - const __m256i r = convolve16_4tap_avx2(ss_256, coeffs); - ss_256[0] = ss_256[1]; - return r; -} - -static inline void xy_y_convolve_4tap_16_avx2(const __m256i *const ss, - const __m256i coeffs[2], - __m256i r[2]) { - r[0] = convolve16_4tap_avx2(ss, coeffs); - r[1] = convolve16_4tap_avx2(ss + 2, coeffs); -} - -static inline void xy_y_convolve_4tap_8x2_avx2(const int16_t *const src, - __m256i ss_256[4], - const __m256i coeffs[2], - __m256i r[2]) { - __m256i s_256[2]; - s_256[0] = _mm256_loadu_si256((__m256i *)(src + 2 * 8)); - s_256[1] = _mm256_loadu_si256((__m256i *)(src + 3 * 8)); - ss_256[1] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r); - ss_256[0] = ss_256[1]; - ss_256[2] = ss_256[3]; -} - -static inline void xy_y_convolve_4tap_8x2_half_pel_avx2( - const int16_t *const src, const __m256i coeffs[1], __m256i s_256[4], - __m256i r[2]) { - __m256i a_256[2]; - s_256[2] = _mm256_loadu_si256((__m256i *)(src + 2 * 8)); - s_256[3] = _mm256_loadu_si256((__m256i *)(src + 3 * 8)); - a_256[0] = _mm256_add_epi16(s_256[0], s_256[3]); - a_256[1] = _mm256_add_epi16(s_256[1], s_256[2]); - xy_y_convolve_2tap_16_avx2(a_256[0], a_256[1], coeffs, r); - s_256[0] = s_256[2]; - s_256[1] = s_256[3]; -} - -static inline void xy_y_convolve_4tap_16x2_avx2( - const int16_t *const src, __m256i s_256[4], __m256i ss_256[4], - __m256i tt_256[4], const __m256i coeffs[2], __m256i r[4]) { - s_256[3] = _mm256_loadu_si256((__m256i *)(src + 3 * 16)); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - s_256[2] = _mm256_loadu_si256((__m256i *)(src + 4 * 16)); - tt_256[1] = _mm256_unpacklo_epi16(s_256[3], s_256[2]); - tt_256[3] = _mm256_unpackhi_epi16(s_256[3], s_256[2]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r + 0); - xy_y_convolve_4tap_16_avx2(tt_256, coeffs, r + 2); - ss_256[0] = ss_256[1]; - ss_256[2] = ss_256[3]; - tt_256[0] = tt_256[1]; - tt_256[2] = tt_256[3]; -} - -static inline void xy_y_convolve_4tap_32x2_avx2( - const int16_t *const src, const ptrdiff_t stride, __m256i s_256[4], - __m256i ss_256[4], __m256i tt_256[4], const __m256i coeffs[2], - __m256i r[4]) { - s_256[3] = _mm256_loadu_si256((__m256i *)(src + 3 * stride)); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - s_256[2] = _mm256_loadu_si256((__m256i *)(src + 4 * stride)); - tt_256[1] = _mm256_unpacklo_epi16(s_256[3], s_256[2]); - tt_256[3] = _mm256_unpackhi_epi16(s_256[3], s_256[2]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r + 0); - xy_y_convolve_4tap_16_avx2(tt_256, coeffs, r + 2); - ss_256[0] = ss_256[1]; - ss_256[2] = ss_256[3]; - tt_256[0] = tt_256[1]; - tt_256[2] = tt_256[3]; -} - -static inline void xy_y_convolve_4tap_16x2_half_pelavx2( - const int16_t *const src, __m256i s_256[5], const __m256i coeffs[1], - __m256i r[4]) { - __m256i a_256[2]; - - s_256[3] = _mm256_loadu_si256((__m256i *)(src + 3 * 16)); - s_256[4] = _mm256_loadu_si256((__m256i *)(src + 4 * 16)); - - a_256[0] = _mm256_add_epi16(s_256[0], s_256[3]); - a_256[1] = _mm256_add_epi16(s_256[1], s_256[2]); - xy_y_convolve_2tap_16_avx2(a_256[0], a_256[1], coeffs, r + 0); - - a_256[0] = _mm256_add_epi16(s_256[1], s_256[4]); - a_256[1] = _mm256_add_epi16(s_256[2], s_256[3]); - xy_y_convolve_2tap_16_avx2(a_256[0], a_256[1], coeffs, r + 2); - - s_256[0] = s_256[2]; - s_256[1] = s_256[3]; - s_256[2] = s_256[4]; -} - -static inline __m128i xy_y_convolve_6tap_2x2_sse2(const int16_t *const src, - __m128i s_32[6], - __m128i ss_128[3], - const __m128i coeffs[3]) { - s_32[5] = _mm_cvtsi32_si128(loadu_int32(src + 5 * 2)); - const __m128i src45 = _mm_unpacklo_epi32(s_32[4], s_32[5]); - s_32[4] = _mm_cvtsi32_si128(loadu_int32(src + 6 * 2)); - const __m128i src56 = _mm_unpacklo_epi32(s_32[5], s_32[4]); - ss_128[2] = _mm_unpacklo_epi16(src45, src56); - const __m128i r = convolve16_6tap_sse2(ss_128, coeffs); - ss_128[0] = ss_128[1]; - ss_128[1] = ss_128[2]; - return r; -} - -static inline __m256i xy_y_convolve_6tap_4x2_avx2(const int16_t *const src, - __m128i s_64[6], - __m256i ss_256[3], - const __m256i coeffs[3]) { - __m256i s_256[2]; - s_64[5] = _mm_loadl_epi64((__m128i *)(src + 5 * 4)); - s_256[0] = _mm256_setr_m128i(s_64[4], s_64[5]); - s_64[4] = _mm_loadl_epi64((__m128i *)(src + 6 * 4)); - s_256[1] = _mm256_setr_m128i(s_64[5], s_64[4]); - ss_256[2] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - const __m256i r = convolve16_6tap_avx2(ss_256, coeffs); - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - return r; -} - -static inline void xy_y_convolve_6tap_16_avx2(const __m256i ss[6], - const __m256i coeffs[3], - __m256i r[2]) { - r[0] = convolve16_6tap_avx2(ss, coeffs); - r[1] = convolve16_6tap_avx2(ss + 3, coeffs); -} - -static inline void xy_y_convolve_6tap_8x2_avx2(const int16_t *const src, - __m256i ss_256[6], - const __m256i coeffs[3], - __m256i r[2]) { - __m256i s_256[2]; - s_256[0] = _mm256_loadu_si256((__m256i *)(src + 4 * 8)); - s_256[1] = _mm256_loadu_si256((__m256i *)(src + 5 * 8)); - ss_256[2] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[5] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - xy_y_convolve_6tap_16_avx2(ss_256, coeffs, r); - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[3] = ss_256[4]; - ss_256[4] = ss_256[5]; -} - -static inline void xy_y_convolve_6tap_8x2_half_pel_avx2( - const int16_t *const src, const __m256i coeffs[2], __m256i s_256[6], - __m256i r[2]) { - __m256i a_256[2], ss_256[4]; - s_256[4] = _mm256_loadu_si256((__m256i *)(src + 4 * 8)); - s_256[5] = _mm256_loadu_si256((__m256i *)(src + 5 * 8)); - a_256[0] = _mm256_add_epi16(s_256[0], s_256[5]); - a_256[1] = _mm256_add_epi16(s_256[1], s_256[4]); - ss_256[0] = _mm256_unpacklo_epi16(a_256[0], a_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[2] = _mm256_unpackhi_epi16(a_256[0], a_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r); - s_256[0] = s_256[2]; - s_256[1] = s_256[3]; - s_256[2] = s_256[4]; - s_256[3] = s_256[5]; -} - -static inline void xy_y_convolve_6tap_16x2_avx2( - const int16_t *const src, const ptrdiff_t stride, __m256i s_256[6], - __m256i ss_256[6], __m256i tt_256[6], const __m256i coeffs[3], - __m256i r[4]) { - s_256[5] = _mm256_loadu_si256((__m256i *)(src + 5 * stride)); - ss_256[2] = _mm256_unpacklo_epi16(s_256[4], s_256[5]); - ss_256[5] = _mm256_unpackhi_epi16(s_256[4], s_256[5]); - s_256[4] = _mm256_loadu_si256((__m256i *)(src + 6 * stride)); - tt_256[2] = _mm256_unpacklo_epi16(s_256[5], s_256[4]); - tt_256[5] = _mm256_unpackhi_epi16(s_256[5], s_256[4]); - - xy_y_convolve_6tap_16_avx2(ss_256, coeffs, r + 0); - xy_y_convolve_6tap_16_avx2(tt_256, coeffs, r + 2); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[3] = ss_256[4]; - ss_256[4] = ss_256[5]; - - tt_256[0] = tt_256[1]; - tt_256[1] = tt_256[2]; - tt_256[3] = tt_256[4]; - tt_256[4] = tt_256[5]; -} - -static inline void xy_y_convolve_6tap_16x2_half_pel_avx2( - const int16_t *const src, const ptrdiff_t stride, __m256i s_256[6], - __m256i ss_256[4], const __m256i coeffs[2], __m256i r[4]) { - __m256i a_256[2]; - - s_256[5] = _mm256_loadu_si256((__m256i *)(src + 5 * stride)); - a_256[0] = _mm256_add_epi16(s_256[0], s_256[5]); - a_256[1] = _mm256_add_epi16(s_256[1], s_256[4]); - ss_256[0] = _mm256_unpacklo_epi16(a_256[0], a_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[2], s_256[3]); - ss_256[2] = _mm256_unpackhi_epi16(a_256[0], a_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[2], s_256[3]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r + 0); - - a_256[1] = _mm256_add_epi16(s_256[2], s_256[5]); - s_256[0] = s_256[2]; - s_256[2] = s_256[4]; - s_256[4] = _mm256_loadu_si256((__m256i *)(src + 6 * stride)); - a_256[0] = _mm256_add_epi16(s_256[1], s_256[4]); - s_256[1] = s_256[3]; - s_256[3] = s_256[5]; - ss_256[0] = _mm256_unpacklo_epi16(a_256[0], a_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(s_256[1], s_256[2]); - ss_256[2] = _mm256_unpackhi_epi16(a_256[0], a_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(s_256[1], s_256[2]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r + 2); -} - -static inline __m128i xy_y_convolve_8tap_2x2_sse2(const int16_t *const src, - __m128i s_32[8], - __m128i ss_128[4], - const __m128i coeffs[4]) { - s_32[7] = _mm_cvtsi32_si128(loadu_int32(src + 7 * 2)); - const __m128i src67 = _mm_unpacklo_epi32(s_32[6], s_32[7]); - s_32[6] = _mm_cvtsi32_si128(loadu_int32(src + 8 * 2)); - const __m128i src78 = _mm_unpacklo_epi32(s_32[7], s_32[6]); - ss_128[3] = _mm_unpacklo_epi16(src67, src78); - const __m128i r = convolve16_8tap_sse2(ss_128, coeffs); - ss_128[0] = ss_128[1]; - ss_128[1] = ss_128[2]; - ss_128[2] = ss_128[3]; - return r; -} - -static inline __m256i xy_y_convolve_8tap_4x2_avx2(const int16_t *const src, - __m128i s_64[8], - __m256i ss_256[4], - const __m256i coeffs[4]) { - __m256i s_256[2]; - s_64[7] = _mm_loadl_epi64((__m128i *)(src + 7 * 4)); - s_256[0] = _mm256_setr_m128i(s_64[6], s_64[7]); - s_64[6] = _mm_loadl_epi64((__m128i *)(src + 8 * 4)); - s_256[1] = _mm256_setr_m128i(s_64[7], s_64[6]); - ss_256[3] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - const __m256i r = convolve16_8tap_avx2(ss_256, coeffs); - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[2] = ss_256[3]; - return r; -} - -static inline void xy_y_convolve_8tap_16_avx2(const __m256i *const ss, - const __m256i coeffs[4], - __m256i r[2]) { - r[0] = convolve16_8tap_avx2(ss, coeffs); - r[1] = convolve16_8tap_avx2(ss + 4, coeffs); -} - -static inline void xy_y_convolve_8tap_8x2_avx2(const int16_t *const src, - __m256i ss_256[8], - const __m256i coeffs[4], - __m256i r[2]) { - __m256i s_256[2]; - s_256[0] = _mm256_loadu_si256((__m256i *)(src + 6 * 8)); - s_256[1] = _mm256_loadu_si256((__m256i *)(src + 7 * 8)); - ss_256[3] = _mm256_unpacklo_epi16(s_256[0], s_256[1]); - ss_256[7] = _mm256_unpackhi_epi16(s_256[0], s_256[1]); - xy_y_convolve_8tap_16_avx2(ss_256, coeffs, r); - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[2] = ss_256[3]; - ss_256[4] = ss_256[5]; - ss_256[5] = ss_256[6]; - ss_256[6] = ss_256[7]; -} - -static inline void xy_y_convolve_8tap_8x2_half_pel_avx2( - const int16_t *const src, const __m256i coeffs[2], __m256i s_256[8], - __m256i r[2]) { - __m256i a_256[4], ss_256[4]; - - s_256[6] = _mm256_loadu_si256((__m256i *)(src + 6 * 8)); - s_256[7] = _mm256_loadu_si256((__m256i *)(src + 7 * 8)); - a_256[0] = _mm256_add_epi16(s_256[0], s_256[7]); - a_256[1] = _mm256_add_epi16(s_256[1], s_256[6]); - a_256[2] = _mm256_add_epi16(s_256[2], s_256[5]); - a_256[3] = _mm256_add_epi16(s_256[3], s_256[4]); - ss_256[0] = _mm256_unpacklo_epi16(a_256[0], a_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(a_256[2], a_256[3]); - ss_256[2] = _mm256_unpackhi_epi16(a_256[0], a_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(a_256[2], a_256[3]); - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r); - s_256[0] = s_256[2]; - s_256[1] = s_256[3]; - s_256[2] = s_256[4]; - s_256[3] = s_256[5]; - s_256[4] = s_256[6]; - s_256[5] = s_256[7]; -} - -static AOM_FORCE_INLINE void xy_y_convolve_8tap_16x2_avx2( - const int16_t *const src, const ptrdiff_t stride, const __m256i coeffs[4], - __m256i s_256[8], __m256i ss_256[8], __m256i tt_256[8], __m256i r[4]) { - s_256[7] = _mm256_loadu_si256((__m256i *)(src + 7 * stride)); - ss_256[3] = _mm256_unpacklo_epi16(s_256[6], s_256[7]); - ss_256[7] = _mm256_unpackhi_epi16(s_256[6], s_256[7]); - s_256[6] = _mm256_loadu_si256((__m256i *)(src + 8 * stride)); - tt_256[3] = _mm256_unpacklo_epi16(s_256[7], s_256[6]); - tt_256[7] = _mm256_unpackhi_epi16(s_256[7], s_256[6]); - - xy_y_convolve_8tap_16_avx2(ss_256, coeffs, r + 0); - xy_y_convolve_8tap_16_avx2(tt_256, coeffs, r + 2); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[2] = ss_256[3]; - ss_256[4] = ss_256[5]; - ss_256[5] = ss_256[6]; - ss_256[6] = ss_256[7]; - - tt_256[0] = tt_256[1]; - tt_256[1] = tt_256[2]; - tt_256[2] = tt_256[3]; - tt_256[4] = tt_256[5]; - tt_256[5] = tt_256[6]; - tt_256[6] = tt_256[7]; -} - -static inline void xy_y_convolve_8tap_16x2_half_pel_avx2( - const int16_t *const src, const ptrdiff_t stride, const __m256i coeffs[4], - __m256i s_256[8], __m256i r[4]) { - __m256i a_256[4], ss_256[4]; - s_256[7] = _mm256_loadu_si256((__m256i *)(src + 7 * stride)); - - a_256[0] = _mm256_add_epi16(s_256[0], s_256[7]); - a_256[1] = _mm256_add_epi16(s_256[1], s_256[6]); - a_256[2] = _mm256_add_epi16(s_256[2], s_256[5]); - a_256[3] = _mm256_add_epi16(s_256[3], s_256[4]); - ss_256[0] = _mm256_unpacklo_epi16(a_256[0], a_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(a_256[2], a_256[3]); - ss_256[2] = _mm256_unpackhi_epi16(a_256[0], a_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(a_256[2], a_256[3]); - - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r + 0); - - a_256[1] = _mm256_add_epi16(s_256[2], s_256[7]); - a_256[2] = _mm256_add_epi16(s_256[3], s_256[6]); - a_256[3] = _mm256_add_epi16(s_256[4], s_256[5]); - s_256[0] = s_256[2]; - s_256[2] = s_256[4]; - s_256[4] = s_256[6]; - s_256[6] = _mm256_loadu_si256((__m256i *)(src + 8 * stride)); - - a_256[0] = _mm256_add_epi16(s_256[1], s_256[6]); - s_256[1] = s_256[3]; - s_256[3] = s_256[5]; - s_256[5] = s_256[7]; - ss_256[0] = _mm256_unpacklo_epi16(a_256[0], a_256[1]); - ss_256[1] = _mm256_unpacklo_epi16(a_256[2], a_256[3]); - ss_256[2] = _mm256_unpackhi_epi16(a_256[0], a_256[1]); - ss_256[3] = _mm256_unpackhi_epi16(a_256[2], a_256[3]); - - xy_y_convolve_4tap_16_avx2(ss_256, coeffs, r + 2); -} - -static inline void xy_y_round_store_8x2_avx2(const __m256i res[2], - uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i r = xy_y_round_16_avx2(res); - pack_store_8x2_avx2(r, dst, stride); -} - -static inline void xy_y_round_store_16x2_avx2(const __m256i res[4], - uint8_t *const dst, - const ptrdiff_t stride) { - const __m256i r0 = xy_y_round_16_avx2(res + 0); - const __m256i r1 = xy_y_round_16_avx2(res + 2); - xy_y_pack_store_16x2_avx2(r0, r1, dst, stride); -} - -static inline void sr_y_round_store_32_avx2(const __m256i res[2], - uint8_t *const dst) { - __m256i r[2]; - - r[0] = sr_y_round_avx2(res[0]); - r[1] = sr_y_round_avx2(res[1]); - convolve_store_32_avx2(r[0], r[1], dst); -} - -static inline void sr_y_round_store_32x2_avx2(const __m256i res[4], - uint8_t *const dst, - const int32_t dst_stride) { - sr_y_round_store_32_avx2(res, dst); - sr_y_round_store_32_avx2(res + 2, dst + dst_stride); -} - -static inline void sr_y_2tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[1], const __m256i s0, - __m256i *const s1, uint8_t *const dst) { - __m256i r[2]; - y_convolve_2tap_32_avx2(src, coeffs, s0, s1, r); - sr_y_round_store_32_avx2(r, dst); -} - -static AOM_FORCE_INLINE void av1_convolve_y_sr_specialized_avx2( - const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, - int32_t w, int32_t h, const InterpFilterParams *filter_params_y, - const int32_t subpel_y_q4) { - int32_t x, y; - __m128i coeffs_128[4]; - __m256i coeffs_256[4]; - - int vert_tap = get_filter_tap(filter_params_y, subpel_y_q4); - - if (vert_tap == 2) { - // vert_filt as 2 tap - const uint8_t *src_ptr = src; - - y = h; - - if (subpel_y_q4 != 8) { - if (w <= 8) { - prepare_half_coeffs_2tap_ssse3(filter_params_y, subpel_y_q4, - coeffs_128); - - if (w == 2) { - __m128i s_16[2]; - - s_16[0] = _mm_cvtsi32_si128(*(int16_t *)src_ptr); - - do { - const __m128i res = y_convolve_2tap_2x2_ssse3(src_ptr, src_stride, - coeffs_128, s_16); - const __m128i r = sr_y_round_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 4) { - __m128i s_32[2]; - - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src_ptr)); - - do { - const __m128i res = y_convolve_2tap_4x2_ssse3(src_ptr, src_stride, - coeffs_128, s_32); - const __m128i r = sr_y_round_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m128i s_64[2], s_128[2]; - - assert(w == 8); - - s_64[0] = _mm_loadl_epi64((__m128i *)src_ptr); - - do { - // Note: Faster than binding to AVX2 registers. - s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + src_stride)); - s_128[0] = _mm_unpacklo_epi64(s_64[0], s_64[1]); - s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride)); - s_128[1] = _mm_unpacklo_epi64(s_64[1], s_64[0]); - const __m128i ss0 = _mm_unpacklo_epi8(s_128[0], s_128[1]); - const __m128i ss1 = _mm_unpackhi_epi8(s_128[0], s_128[1]); - const __m128i res0 = convolve_2tap_ssse3(&ss0, coeffs_128); - const __m128i res1 = convolve_2tap_ssse3(&ss1, coeffs_128); - const __m128i r0 = sr_y_round_sse2(res0); - const __m128i r1 = sr_y_round_sse2(res1); - const __m128i d = _mm_packus_epi16(r0, r1); - _mm_storel_epi64((__m128i *)dst, d); - _mm_storeh_epi64((__m128i *)(dst + dst_stride), d); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - prepare_half_coeffs_2tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 16) { - __m128i s_128[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)src_ptr); - - do { - __m256i r[2]; - - y_convolve_2tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, - r); - sr_y_round_store_16x2_avx2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - __m256i s_256[2]; - - s_256[0] = _mm256_loadu_si256((__m256i *)src_ptr); - - do { - sr_y_2tap_32_avx2(src_ptr + src_stride, coeffs_256, s_256[0], - &s_256[1], dst); - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride, coeffs_256, s_256[1], - &s_256[0], dst + dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 64) { - __m256i s_256[2][2]; - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * 32)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * 32)); - - do { - sr_y_2tap_32_avx2(src_ptr + src_stride, coeffs_256, s_256[0][0], - &s_256[1][0], dst); - sr_y_2tap_32_avx2(src_ptr + src_stride + 32, coeffs_256, - s_256[0][1], &s_256[1][1], dst + 32); - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride, coeffs_256, s_256[1][0], - &s_256[0][0], dst + dst_stride); - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride + 32, coeffs_256, - s_256[1][1], &s_256[0][1], dst + dst_stride + 32); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i s_256[2][4]; - - assert(w == 128); - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * 32)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * 32)); - s_256[0][2] = _mm256_loadu_si256((__m256i *)(src_ptr + 2 * 32)); - s_256[0][3] = _mm256_loadu_si256((__m256i *)(src_ptr + 3 * 32)); - - do { - sr_y_2tap_32_avx2(src_ptr + src_stride, coeffs_256, s_256[0][0], - &s_256[1][0], dst); - sr_y_2tap_32_avx2(src_ptr + src_stride + 1 * 32, coeffs_256, - s_256[0][1], &s_256[1][1], dst + 1 * 32); - sr_y_2tap_32_avx2(src_ptr + src_stride + 2 * 32, coeffs_256, - s_256[0][2], &s_256[1][2], dst + 2 * 32); - sr_y_2tap_32_avx2(src_ptr + src_stride + 3 * 32, coeffs_256, - s_256[0][3], &s_256[1][3], dst + 3 * 32); - - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride, coeffs_256, s_256[1][0], - &s_256[0][0], dst + dst_stride); - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride + 1 * 32, coeffs_256, - s_256[1][1], &s_256[0][1], - dst + dst_stride + 1 * 32); - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride + 2 * 32, coeffs_256, - s_256[1][2], &s_256[0][2], - dst + dst_stride + 2 * 32); - sr_y_2tap_32_avx2(src_ptr + 2 * src_stride + 3 * 32, coeffs_256, - s_256[1][3], &s_256[0][3], - dst + dst_stride + 3 * 32); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } - } else { - // average to get half pel - if (w <= 8) { - if (w == 2) { - __m128i s_16[2]; - - s_16[0] = _mm_cvtsi32_si128(*(int16_t *)src_ptr); - - do { - s_16[1] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + src_stride)); - const __m128i d0 = _mm_avg_epu8(s_16[0], s_16[1]); - *(int16_t *)dst = (int16_t)_mm_cvtsi128_si32(d0); - s_16[0] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 2 * src_stride)); - const __m128i d1 = _mm_avg_epu8(s_16[1], s_16[0]); - *(int16_t *)(dst + dst_stride) = (int16_t)_mm_cvtsi128_si32(d1); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 4) { - __m128i s_32[2]; - - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src_ptr)); - - do { - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src_ptr + src_stride)); - const __m128i d0 = _mm_avg_epu8(s_32[0], s_32[1]); - xx_storel_32(dst, d0); - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 2 * src_stride)); - const __m128i d1 = _mm_avg_epu8(s_32[1], s_32[0]); - xx_storel_32(dst + dst_stride, d1); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m128i s_64[2]; - - assert(w == 8); - - s_64[0] = _mm_loadl_epi64((__m128i *)src_ptr); - - do { - // Note: Faster than binding to AVX2 registers. - s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + src_stride)); - const __m128i d0 = _mm_avg_epu8(s_64[0], s_64[1]); - _mm_storel_epi64((__m128i *)dst, d0); - s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride)); - const __m128i d1 = _mm_avg_epu8(s_64[1], s_64[0]); - _mm_storel_epi64((__m128i *)(dst + dst_stride), d1); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else if (w == 16) { - __m128i s_128[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)src_ptr); - - do { - s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + src_stride)); - const __m128i d0 = _mm_avg_epu8(s_128[0], s_128[1]); - _mm_storeu_si128((__m128i *)dst, d0); - s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride)); - const __m128i d1 = _mm_avg_epu8(s_128[1], s_128[0]); - _mm_storeu_si128((__m128i *)(dst + dst_stride), d1); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - __m256i s_256[2]; - - s_256[0] = _mm256_loadu_si256((__m256i *)src_ptr); - - do { - sr_y_2tap_32_avg_avx2(src_ptr + src_stride, s_256[0], &s_256[1], dst); - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride, s_256[1], &s_256[0], - dst + dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 64) { - __m256i s_256[2][2]; - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * 32)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * 32)); - - do { - sr_y_2tap_32_avg_avx2(src_ptr + src_stride, s_256[0][0], &s_256[1][0], - dst); - sr_y_2tap_32_avg_avx2(src_ptr + src_stride + 32, s_256[0][1], - &s_256[1][1], dst + 32); - - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride, s_256[1][0], - &s_256[0][0], dst + dst_stride); - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride + 32, s_256[1][1], - &s_256[0][1], dst + dst_stride + 32); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i s_256[2][4]; - - assert(w == 128); - - s_256[0][0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * 32)); - s_256[0][1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * 32)); - s_256[0][2] = _mm256_loadu_si256((__m256i *)(src_ptr + 2 * 32)); - s_256[0][3] = _mm256_loadu_si256((__m256i *)(src_ptr + 3 * 32)); - - do { - sr_y_2tap_32_avg_avx2(src_ptr + src_stride, s_256[0][0], &s_256[1][0], - dst); - sr_y_2tap_32_avg_avx2(src_ptr + src_stride + 1 * 32, s_256[0][1], - &s_256[1][1], dst + 1 * 32); - sr_y_2tap_32_avg_avx2(src_ptr + src_stride + 2 * 32, s_256[0][2], - &s_256[1][2], dst + 2 * 32); - sr_y_2tap_32_avg_avx2(src_ptr + src_stride + 3 * 32, s_256[0][3], - &s_256[1][3], dst + 3 * 32); - - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride, s_256[1][0], - &s_256[0][0], dst + dst_stride); - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride + 1 * 32, s_256[1][1], - &s_256[0][1], dst + dst_stride + 1 * 32); - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride + 2 * 32, s_256[1][2], - &s_256[0][2], dst + dst_stride + 2 * 32); - sr_y_2tap_32_avg_avx2(src_ptr + 2 * src_stride + 3 * 32, s_256[1][3], - &s_256[0][3], dst + dst_stride + 3 * 32); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } - } else if (vert_tap == 4) { - // vert_filt as 4 tap - const uint8_t *src_ptr = src - src_stride; - - y = h; - - if (w <= 4) { - prepare_half_coeffs_4tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128); - - if (w == 2) { - __m128i s_16[4], ss_128[2]; - - s_16[0] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 0 * src_stride)); - s_16[1] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 1 * src_stride)); - s_16[2] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 2 * src_stride)); - - const __m128i src01 = _mm_unpacklo_epi16(s_16[0], s_16[1]); - const __m128i src12 = _mm_unpacklo_epi16(s_16[1], s_16[2]); - - ss_128[0] = _mm_unpacklo_epi8(src01, src12); - - do { - src_ptr += 2 * src_stride; - const __m128i res = y_convolve_4tap_2x2_ssse3( - src_ptr, src_stride, coeffs_128, s_16, ss_128); - const __m128i r = sr_y_round_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - - ss_128[0] = ss_128[1]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m128i s_32[4], ss_128[2]; - - assert(w == 4); - - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 0 * src_stride)); - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 1 * src_stride)); - s_32[2] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 2 * src_stride)); - - const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]); - const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]); - - ss_128[0] = _mm_unpacklo_epi8(src01, src12); - - do { - src_ptr += 2 * src_stride; - const __m128i res = y_convolve_4tap_4x2_ssse3( - src_ptr, src_stride, coeffs_128, s_32, ss_128); - const __m128i r = sr_y_round_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - - ss_128[0] = ss_128[1]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - prepare_half_coeffs_4tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 8) { - __m128i s_64[4]; - __m256i ss_256[2]; - - s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 0 * src_stride)); - s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + 1 * src_stride)); - s_64[2] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src01 = _mm256_setr_m128i(s_64[0], s_64[1]); - const __m256i src12 = _mm256_setr_m128i(s_64[1], s_64[2]); - - ss_256[0] = _mm256_unpacklo_epi8(src01, src12); - - do { - src_ptr += 2 * src_stride; - const __m256i res = y_convolve_4tap_8x2_avx2( - src_ptr, src_stride, coeffs_256, s_64, ss_256); - sr_y_round_store_8x2_avx2(res, dst, dst_stride); - - ss_256[0] = ss_256[1]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - __m128i s_128[4]; - __m256i ss_256[4], r[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 0 * src_stride)); - s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + 1 * src_stride)); - s_128[2] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src01 = _mm256_setr_m128i(s_128[0], s_128[1]); - const __m256i src12 = _mm256_setr_m128i(s_128[1], s_128[2]); - - ss_256[0] = _mm256_unpacklo_epi8(src01, src12); - ss_256[2] = _mm256_unpackhi_epi8(src01, src12); - - do { - src_ptr += 2 * src_stride; - y_convolve_4tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, - ss_256, r); - sr_y_round_store_16x2_avx2(r, dst, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[2] = ss_256[3]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - // AV1 standard won't have 32x4 case. - // This only favors some optimization feature which - // subsamples 32x8 to 32x4 and triggers 4-tap filter. - - __m256i s_256[4], ss_256[4], tt_256[4], r[4]; - - s_256[0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * src_stride)); - s_256[1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * src_stride)); - s_256[2] = _mm256_loadu_si256((__m256i *)(src_ptr + 2 * src_stride)); - - ss_256[0] = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - ss_256[2] = _mm256_unpackhi_epi8(s_256[0], s_256[1]); - - tt_256[0] = _mm256_unpacklo_epi8(s_256[1], s_256[2]); - tt_256[2] = _mm256_unpackhi_epi8(s_256[1], s_256[2]); - - do { - src_ptr += 2 * src_stride; - y_convolve_4tap_32x2_avx2(src_ptr, src_stride, coeffs_256, s_256, - ss_256, tt_256, r); - sr_y_round_store_32x2_avx2(r, dst, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[2] = ss_256[3]; - - tt_256[0] = tt_256[1]; - tt_256[2] = tt_256[3]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - assert(!(w % 32)); - - __m256i s_256[4], ss_256[4], tt_256[4], r[4]; - x = 0; - do { - const uint8_t *s = src_ptr + x; - uint8_t *d = dst + x; - s_256[0] = _mm256_loadu_si256((__m256i *)(s + 0 * src_stride)); - s_256[1] = _mm256_loadu_si256((__m256i *)(s + 1 * src_stride)); - s_256[2] = _mm256_loadu_si256((__m256i *)(s + 2 * src_stride)); - - ss_256[0] = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - ss_256[2] = _mm256_unpackhi_epi8(s_256[0], s_256[1]); - - tt_256[0] = _mm256_unpacklo_epi8(s_256[1], s_256[2]); - tt_256[2] = _mm256_unpackhi_epi8(s_256[1], s_256[2]); - - y = h; - do { - s += 2 * src_stride; - y_convolve_4tap_32x2_avx2(s, src_stride, coeffs_256, s_256, ss_256, - tt_256, r); - sr_y_round_store_32x2_avx2(r, d, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[2] = ss_256[3]; - - tt_256[0] = tt_256[1]; - tt_256[2] = tt_256[3]; - d += 2 * dst_stride; - y -= 2; - } while (y); - x += 32; - } while (x < w); - } - } - } else if (vert_tap == 6) { - // vert_filt as 6 tap - const uint8_t *src_ptr = src - 2 * src_stride; - - if (w <= 4) { - prepare_half_coeffs_6tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128); - - y = h; - - if (w == 2) { - __m128i s_16[6], ss_128[3]; - - s_16[0] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 0 * src_stride)); - s_16[1] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 1 * src_stride)); - s_16[2] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 2 * src_stride)); - s_16[3] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 3 * src_stride)); - s_16[4] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 4 * src_stride)); - - const __m128i src01 = _mm_unpacklo_epi16(s_16[0], s_16[1]); - const __m128i src12 = _mm_unpacklo_epi16(s_16[1], s_16[2]); - const __m128i src23 = _mm_unpacklo_epi16(s_16[2], s_16[3]); - const __m128i src34 = _mm_unpacklo_epi16(s_16[3], s_16[4]); - - ss_128[0] = _mm_unpacklo_epi8(src01, src12); - ss_128[1] = _mm_unpacklo_epi8(src23, src34); - - do { - src_ptr += 2 * src_stride; - const __m128i res = y_convolve_6tap_2x2_ssse3( - src_ptr, src_stride, coeffs_128, s_16, ss_128); - const __m128i r = sr_y_round_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - - ss_128[0] = ss_128[1]; - ss_128[1] = ss_128[2]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m128i s_32[6], ss_128[3]; - - assert(w == 4); - - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 0 * src_stride)); - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 1 * src_stride)); - s_32[2] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 2 * src_stride)); - s_32[3] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 3 * src_stride)); - s_32[4] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 4 * src_stride)); - - const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]); - const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]); - const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]); - const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[4]); - - ss_128[0] = _mm_unpacklo_epi8(src01, src12); - ss_128[1] = _mm_unpacklo_epi8(src23, src34); - - do { - src_ptr += 2 * src_stride; - const __m128i res = y_convolve_6tap_4x2_ssse3( - src_ptr, src_stride, coeffs_128, s_32, ss_128); - const __m128i r = sr_y_round_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - - ss_128[0] = ss_128[1]; - ss_128[1] = ss_128[2]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - prepare_half_coeffs_6tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 8) { - __m128i s_64[6]; - __m256i ss_256[3]; - - s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 0 * src_stride)); - s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + 1 * src_stride)); - s_64[2] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride)); - s_64[3] = _mm_loadl_epi64((__m128i *)(src_ptr + 3 * src_stride)); - s_64[4] = _mm_loadl_epi64((__m128i *)(src_ptr + 4 * src_stride)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src01 = _mm256_setr_m128i(s_64[0], s_64[1]); - const __m256i src12 = _mm256_setr_m128i(s_64[1], s_64[2]); - const __m256i src23 = _mm256_setr_m128i(s_64[2], s_64[3]); - const __m256i src34 = _mm256_setr_m128i(s_64[3], s_64[4]); - - ss_256[0] = _mm256_unpacklo_epi8(src01, src12); - ss_256[1] = _mm256_unpacklo_epi8(src23, src34); - - y = h; - do { - src_ptr += 2 * src_stride; - const __m256i res = y_convolve_6tap_8x2_avx2( - src_ptr, src_stride, coeffs_256, s_64, ss_256); - sr_y_round_store_8x2_avx2(res, dst, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - __m128i s_128[6]; - __m256i ss_256[6], r[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 0 * src_stride)); - s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + 1 * src_stride)); - s_128[2] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride)); - s_128[3] = _mm_loadu_si128((__m128i *)(src_ptr + 3 * src_stride)); - s_128[4] = _mm_loadu_si128((__m128i *)(src_ptr + 4 * src_stride)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src01 = _mm256_setr_m128i(s_128[0], s_128[1]); - const __m256i src12 = _mm256_setr_m128i(s_128[1], s_128[2]); - const __m256i src23 = _mm256_setr_m128i(s_128[2], s_128[3]); - const __m256i src34 = _mm256_setr_m128i(s_128[3], s_128[4]); - - ss_256[0] = _mm256_unpacklo_epi8(src01, src12); - ss_256[1] = _mm256_unpacklo_epi8(src23, src34); - - ss_256[3] = _mm256_unpackhi_epi8(src01, src12); - ss_256[4] = _mm256_unpackhi_epi8(src23, src34); - - y = h; - do { - src_ptr += 2 * src_stride; - y_convolve_6tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, - ss_256, r); - sr_y_round_store_16x2_avx2(r, dst, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - - ss_256[3] = ss_256[4]; - ss_256[4] = ss_256[5]; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i s_256[6], ss_256[6], tt_256[6], r[4]; - - assert(!(w % 32)); - - x = 0; - do { - const uint8_t *s = src_ptr + x; - uint8_t *d = dst + x; - - s_256[0] = _mm256_loadu_si256((__m256i *)(s + 0 * src_stride)); - s_256[1] = _mm256_loadu_si256((__m256i *)(s + 1 * src_stride)); - s_256[2] = _mm256_loadu_si256((__m256i *)(s + 2 * src_stride)); - s_256[3] = _mm256_loadu_si256((__m256i *)(s + 3 * src_stride)); - s_256[4] = _mm256_loadu_si256((__m256i *)(s + 4 * src_stride)); - - ss_256[0] = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi8(s_256[2], s_256[3]); - ss_256[3] = _mm256_unpackhi_epi8(s_256[0], s_256[1]); - ss_256[4] = _mm256_unpackhi_epi8(s_256[2], s_256[3]); - - tt_256[0] = _mm256_unpacklo_epi8(s_256[1], s_256[2]); - tt_256[1] = _mm256_unpacklo_epi8(s_256[3], s_256[4]); - tt_256[3] = _mm256_unpackhi_epi8(s_256[1], s_256[2]); - tt_256[4] = _mm256_unpackhi_epi8(s_256[3], s_256[4]); - - y = h; - do { - s += 2 * src_stride; - y_convolve_6tap_32x2_avx2(s, src_stride, coeffs_256, s_256, ss_256, - tt_256, r); - sr_y_round_store_32x2_avx2(r, d, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[3] = ss_256[4]; - ss_256[4] = ss_256[5]; - - tt_256[0] = tt_256[1]; - tt_256[1] = tt_256[2]; - tt_256[3] = tt_256[4]; - tt_256[4] = tt_256[5]; - d += 2 * dst_stride; - y -= 2; - } while (y); - - x += 32; - } while (x < w); - } - } - } else if (vert_tap == 8) { - // vert_filt as 8 tap - const uint8_t *src_ptr = src - 3 * src_stride; - - if (w <= 4) { - prepare_half_coeffs_8tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128); - - y = h; - - if (w == 2) { - __m128i s_16[8], ss_128[4]; - - s_16[0] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 0 * src_stride)); - s_16[1] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 1 * src_stride)); - s_16[2] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 2 * src_stride)); - s_16[3] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 3 * src_stride)); - s_16[4] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 4 * src_stride)); - s_16[5] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 5 * src_stride)); - s_16[6] = _mm_cvtsi32_si128(loadu_int16(src_ptr + 6 * src_stride)); - - const __m128i src01 = _mm_unpacklo_epi16(s_16[0], s_16[1]); - const __m128i src12 = _mm_unpacklo_epi16(s_16[1], s_16[2]); - const __m128i src23 = _mm_unpacklo_epi16(s_16[2], s_16[3]); - const __m128i src34 = _mm_unpacklo_epi16(s_16[3], s_16[4]); - const __m128i src45 = _mm_unpacklo_epi16(s_16[4], s_16[5]); - const __m128i src56 = _mm_unpacklo_epi16(s_16[5], s_16[6]); - - ss_128[0] = _mm_unpacklo_epi8(src01, src12); - ss_128[1] = _mm_unpacklo_epi8(src23, src34); - ss_128[2] = _mm_unpacklo_epi8(src45, src56); - - do { - const __m128i res = y_convolve_8tap_2x2_ssse3( - src_ptr, src_stride, coeffs_128, s_16, ss_128); - const __m128i r = sr_y_round_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - ss_128[0] = ss_128[1]; - ss_128[1] = ss_128[2]; - ss_128[2] = ss_128[3]; - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m128i s_32[8], ss_128[4]; - - assert(w == 4); - - s_32[0] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 0 * src_stride)); - s_32[1] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 1 * src_stride)); - s_32[2] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 2 * src_stride)); - s_32[3] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 3 * src_stride)); - s_32[4] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 4 * src_stride)); - s_32[5] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 5 * src_stride)); - s_32[6] = _mm_cvtsi32_si128(loadu_int32(src_ptr + 6 * src_stride)); - - const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]); - const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]); - const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]); - const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[4]); - const __m128i src45 = _mm_unpacklo_epi32(s_32[4], s_32[5]); - const __m128i src56 = _mm_unpacklo_epi32(s_32[5], s_32[6]); - - ss_128[0] = _mm_unpacklo_epi8(src01, src12); - ss_128[1] = _mm_unpacklo_epi8(src23, src34); - ss_128[2] = _mm_unpacklo_epi8(src45, src56); - - do { - const __m128i res = y_convolve_8tap_4x2_ssse3( - src_ptr, src_stride, coeffs_128, s_32, ss_128); - const __m128i r = sr_y_round_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - ss_128[0] = ss_128[1]; - ss_128[1] = ss_128[2]; - ss_128[2] = ss_128[3]; - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - prepare_half_coeffs_8tap_avx2(filter_params_y, subpel_y_q4, coeffs_256); - - if (w == 8) { - __m128i s_64[8]; - __m256i ss_256[4]; - - s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 0 * src_stride)); - s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + 1 * src_stride)); - s_64[2] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride)); - s_64[3] = _mm_loadl_epi64((__m128i *)(src_ptr + 3 * src_stride)); - s_64[4] = _mm_loadl_epi64((__m128i *)(src_ptr + 4 * src_stride)); - s_64[5] = _mm_loadl_epi64((__m128i *)(src_ptr + 5 * src_stride)); - s_64[6] = _mm_loadl_epi64((__m128i *)(src_ptr + 6 * src_stride)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src01 = _mm256_setr_m128i(s_64[0], s_64[1]); - const __m256i src12 = _mm256_setr_m128i(s_64[1], s_64[2]); - const __m256i src23 = _mm256_setr_m128i(s_64[2], s_64[3]); - const __m256i src34 = _mm256_setr_m128i(s_64[3], s_64[4]); - const __m256i src45 = _mm256_setr_m128i(s_64[4], s_64[5]); - const __m256i src56 = _mm256_setr_m128i(s_64[5], s_64[6]); - - ss_256[0] = _mm256_unpacklo_epi8(src01, src12); - ss_256[1] = _mm256_unpacklo_epi8(src23, src34); - ss_256[2] = _mm256_unpacklo_epi8(src45, src56); - - y = h; - do { - const __m256i res = y_convolve_8tap_8x2_avx2( - src_ptr, src_stride, coeffs_256, s_64, ss_256); - sr_y_round_store_8x2_avx2(res, dst, dst_stride); - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[2] = ss_256[3]; - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - __m128i s_128[8]; - __m256i ss_256[8], r[2]; - - s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 0 * src_stride)); - s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + 1 * src_stride)); - s_128[2] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride)); - s_128[3] = _mm_loadu_si128((__m128i *)(src_ptr + 3 * src_stride)); - s_128[4] = _mm_loadu_si128((__m128i *)(src_ptr + 4 * src_stride)); - s_128[5] = _mm_loadu_si128((__m128i *)(src_ptr + 5 * src_stride)); - s_128[6] = _mm_loadu_si128((__m128i *)(src_ptr + 6 * src_stride)); - - // Load lines a and b. Line a to lower 128, line b to upper 128 - const __m256i src01 = _mm256_setr_m128i(s_128[0], s_128[1]); - const __m256i src12 = _mm256_setr_m128i(s_128[1], s_128[2]); - const __m256i src23 = _mm256_setr_m128i(s_128[2], s_128[3]); - const __m256i src34 = _mm256_setr_m128i(s_128[3], s_128[4]); - const __m256i src45 = _mm256_setr_m128i(s_128[4], s_128[5]); - const __m256i src56 = _mm256_setr_m128i(s_128[5], s_128[6]); - - ss_256[0] = _mm256_unpacklo_epi8(src01, src12); - ss_256[1] = _mm256_unpacklo_epi8(src23, src34); - ss_256[2] = _mm256_unpacklo_epi8(src45, src56); - - ss_256[4] = _mm256_unpackhi_epi8(src01, src12); - ss_256[5] = _mm256_unpackhi_epi8(src23, src34); - ss_256[6] = _mm256_unpackhi_epi8(src45, src56); - - y = h; - do { - y_convolve_8tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, - ss_256, r); - sr_y_round_store_16x2_avx2(r, dst, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[2] = ss_256[3]; - - ss_256[4] = ss_256[5]; - ss_256[5] = ss_256[6]; - ss_256[6] = ss_256[7]; - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - __m256i s_256[8], ss_256[8], tt_256[8], r[4]; - - assert(!(w % 32)); - - x = 0; - do { - const uint8_t *s = src_ptr + x; - uint8_t *d = dst + x; - - s_256[0] = _mm256_loadu_si256((__m256i *)(s + 0 * src_stride)); - s_256[1] = _mm256_loadu_si256((__m256i *)(s + 1 * src_stride)); - s_256[2] = _mm256_loadu_si256((__m256i *)(s + 2 * src_stride)); - s_256[3] = _mm256_loadu_si256((__m256i *)(s + 3 * src_stride)); - s_256[4] = _mm256_loadu_si256((__m256i *)(s + 4 * src_stride)); - s_256[5] = _mm256_loadu_si256((__m256i *)(s + 5 * src_stride)); - s_256[6] = _mm256_loadu_si256((__m256i *)(s + 6 * src_stride)); - - ss_256[0] = _mm256_unpacklo_epi8(s_256[0], s_256[1]); - ss_256[1] = _mm256_unpacklo_epi8(s_256[2], s_256[3]); - ss_256[2] = _mm256_unpacklo_epi8(s_256[4], s_256[5]); - ss_256[4] = _mm256_unpackhi_epi8(s_256[0], s_256[1]); - ss_256[5] = _mm256_unpackhi_epi8(s_256[2], s_256[3]); - ss_256[6] = _mm256_unpackhi_epi8(s_256[4], s_256[5]); - - tt_256[0] = _mm256_unpacklo_epi8(s_256[1], s_256[2]); - tt_256[1] = _mm256_unpacklo_epi8(s_256[3], s_256[4]); - tt_256[2] = _mm256_unpacklo_epi8(s_256[5], s_256[6]); - tt_256[4] = _mm256_unpackhi_epi8(s_256[1], s_256[2]); - tt_256[5] = _mm256_unpackhi_epi8(s_256[3], s_256[4]); - tt_256[6] = _mm256_unpackhi_epi8(s_256[5], s_256[6]); - - y = h; - do { - y_convolve_8tap_32x2_avx2(s, src_stride, coeffs_256, s_256, ss_256, - tt_256, r); - sr_y_round_store_32x2_avx2(r, d, dst_stride); - - ss_256[0] = ss_256[1]; - ss_256[1] = ss_256[2]; - ss_256[2] = ss_256[3]; - ss_256[4] = ss_256[5]; - ss_256[5] = ss_256[6]; - ss_256[6] = ss_256[7]; - - tt_256[0] = tt_256[1]; - tt_256[1] = tt_256[2]; - tt_256[2] = tt_256[3]; - tt_256[4] = tt_256[5]; - tt_256[5] = tt_256[6]; - tt_256[6] = tt_256[7]; - s += 2 * src_stride; - d += 2 * dst_stride; - y -= 2; - } while (y); - - x += 32; - } while (x < w); - } - } - } -} - -static inline void sr_x_2tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[1], - uint8_t *const dst) { - __m256i r[2]; - - x_convolve_2tap_32_avx2(src, coeffs, r); - sr_x_round_store_32_avx2(r, dst); -} - -static inline void sr_x_6tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[3], - const __m256i filt[3], - uint8_t *const dst) { - __m256i r[2]; - - x_convolve_6tap_32_avx2(src, coeffs, filt, r); - sr_x_round_store_32_avx2(r, dst); -} - -static AOM_FORCE_INLINE void sr_x_8tap_32_avx2(const uint8_t *const src, - const __m256i coeffs[4], - const __m256i filt[4], - uint8_t *const dst) { - __m256i r[2]; - - x_convolve_8tap_32_avx2(src, coeffs, filt, r); - sr_x_round_store_32_avx2(r, dst); -} - -static AOM_FORCE_INLINE void av1_convolve_x_sr_specialized_avx2( - const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, - int32_t w, int32_t h, const InterpFilterParams *filter_params_x, - const int32_t subpel_x_q4, ConvolveParams *conv_params) { - int32_t y = h; - __m128i coeffs_128[4]; - __m256i coeffs_256[4]; - - assert(conv_params->round_0 == 3); - assert((FILTER_BITS - conv_params->round_1) >= 0 || - ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); - (void)conv_params; - - const int horz_tap = get_filter_tap(filter_params_x, subpel_x_q4); - - if (horz_tap == 2) { - // horz_filt as 2 tap - const uint8_t *src_ptr = src; - - if (subpel_x_q4 != 8) { - if (w <= 8) { - prepare_half_coeffs_2tap_ssse3(filter_params_x, subpel_x_q4, - coeffs_128); - - if (w == 2) { - do { - const __m128i res = - x_convolve_2tap_2x2_sse4_1(src_ptr, src_stride, coeffs_128); - const __m128i r = sr_x_round_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 4) { - do { - const __m128i res = - x_convolve_2tap_4x2_ssse3(src_ptr, src_stride, coeffs_128); - const __m128i r = sr_x_round_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else { - assert(w == 8); - - do { - __m128i res[2]; - - x_convolve_2tap_8x2_ssse3(src_ptr, src_stride, coeffs_128, res); - res[0] = sr_x_round_sse2(res[0]); - res[1] = sr_x_round_sse2(res[1]); - const __m128i d = _mm_packus_epi16(res[0], res[1]); - _mm_storel_epi64((__m128i *)dst, d); - _mm_storeh_epi64((__m128i *)(dst + dst_stride), d); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } - } else { - prepare_half_coeffs_2tap_avx2(filter_params_x, subpel_x_q4, coeffs_256); - - if (w == 16) { - do { - __m256i r[2]; - - x_convolve_2tap_16x2_avx2(src_ptr, src_stride, coeffs_256, r); - sr_x_round_store_16x2_avx2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - do { - sr_x_2tap_32_avx2(src_ptr, coeffs_256, dst); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else if (w == 64) { - do { - sr_x_2tap_32_avx2(src_ptr + 0 * 32, coeffs_256, dst + 0 * 32); - sr_x_2tap_32_avx2(src_ptr + 1 * 32, coeffs_256, dst + 1 * 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else { - assert(w == 128); - - do { - sr_x_2tap_32_avx2(src_ptr + 0 * 32, coeffs_256, dst + 0 * 32); - sr_x_2tap_32_avx2(src_ptr + 1 * 32, coeffs_256, dst + 1 * 32); - sr_x_2tap_32_avx2(src_ptr + 2 * 32, coeffs_256, dst + 2 * 32); - sr_x_2tap_32_avx2(src_ptr + 3 * 32, coeffs_256, dst + 3 * 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } - } - } else { - // average to get half pel - if (w == 2) { - do { - __m128i s_128; - - s_128 = load_u8_4x2_sse4_1(src_ptr, src_stride); - const __m128i s1 = _mm_srli_si128(s_128, 1); - const __m128i d = _mm_avg_epu8(s_128, s1); - *(uint16_t *)dst = (uint16_t)_mm_cvtsi128_si32(d); - *(uint16_t *)(dst + dst_stride) = _mm_extract_epi16(d, 2); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 4) { - do { - __m128i s_128; - - s_128 = load_u8_8x2_sse2(src_ptr, src_stride); - const __m128i s1 = _mm_srli_si128(s_128, 1); - const __m128i d = _mm_avg_epu8(s_128, s1); - xx_storel_32(dst, d); - *(int32_t *)(dst + dst_stride) = _mm_extract_epi32(d, 2); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 8) { - do { - const __m128i s00 = _mm_loadu_si128((__m128i *)src_ptr); - const __m128i s10 = - _mm_loadu_si128((__m128i *)(src_ptr + src_stride)); - const __m128i s01 = _mm_srli_si128(s00, 1); - const __m128i s11 = _mm_srli_si128(s10, 1); - const __m128i d0 = _mm_avg_epu8(s00, s01); - const __m128i d1 = _mm_avg_epu8(s10, s11); - _mm_storel_epi64((__m128i *)dst, d0); - _mm_storel_epi64((__m128i *)(dst + dst_stride), d1); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - do { - const __m128i s00 = _mm_loadu_si128((__m128i *)src_ptr); - const __m128i s01 = _mm_loadu_si128((__m128i *)(src_ptr + 1)); - const __m128i s10 = - _mm_loadu_si128((__m128i *)(src_ptr + src_stride)); - const __m128i s11 = - _mm_loadu_si128((__m128i *)(src_ptr + src_stride + 1)); - const __m128i d0 = _mm_avg_epu8(s00, s01); - const __m128i d1 = _mm_avg_epu8(s10, s11); - _mm_storeu_si128((__m128i *)dst, d0); - _mm_storeu_si128((__m128i *)(dst + dst_stride), d1); - - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - do { - sr_x_2tap_32_avg_avx2(src_ptr, dst); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else if (w == 64) { - do { - sr_x_2tap_32_avg_avx2(src_ptr + 0 * 32, dst + 0 * 32); - sr_x_2tap_32_avg_avx2(src_ptr + 1 * 32, dst + 1 * 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else { - assert(w == 128); - - do { - sr_x_2tap_32_avg_avx2(src_ptr + 0 * 32, dst + 0 * 32); - sr_x_2tap_32_avg_avx2(src_ptr + 1 * 32, dst + 1 * 32); - sr_x_2tap_32_avg_avx2(src_ptr + 2 * 32, dst + 2 * 32); - sr_x_2tap_32_avg_avx2(src_ptr + 3 * 32, dst + 3 * 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } - } - } else if (horz_tap == 4) { - // horz_filt as 4 tap - const uint8_t *src_ptr = src - 1; - - prepare_half_coeffs_4tap_ssse3(filter_params_x, subpel_x_q4, coeffs_128); - - if (w == 2) { - do { - const __m128i res = - x_convolve_4tap_2x2_ssse3(src_ptr, src_stride, coeffs_128); - const __m128i r = sr_x_round_sse2(res); - pack_store_2x2_sse2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 4) { - do { - const __m128i res = - x_convolve_4tap_4x2_ssse3(src_ptr, src_stride, coeffs_128); - const __m128i r = sr_x_round_sse2(res); - pack_store_4x2_sse2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 8) { - // TODO(chiyotsai@google.com): Reuse the old SIMD code here. Need to - // rewrite this for better performance later. - __m256i filt_256[2]; - prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs_256); - - filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx2); - filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx2); - for (int i = 0; i < h; i += 2) { - const __m256i data = _mm256_permute2x128_si256( - _mm256_castsi128_si256( - _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride]))), - _mm256_castsi128_si256(_mm_loadu_si128( - (__m128i *)(&src_ptr[i * src_stride + src_stride]))), - 0x20); - - __m256i res_16b = convolve_lowbd_x_4tap(data, coeffs_256 + 1, filt_256); - res_16b = sr_x_round_avx2(res_16b); - - __m256i res_8b = _mm256_packus_epi16(res_16b, res_16b); - - const __m128i res_0 = _mm256_castsi256_si128(res_8b); - const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); - - _mm_storel_epi64((__m128i *)&dst[i * dst_stride], res_0); - _mm_storel_epi64((__m128i *)&dst[i * dst_stride + dst_stride], res_1); - } - } else { - assert(!(w % 16)); - // TODO(chiyotsai@google.com): Reuse the old SIMD code here. Need to - // rewrite this for better performance later. - __m256i filt_256[2]; - prepare_coeffs_lowbd(filter_params_x, subpel_x_q4, coeffs_256); - filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx2); - filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx2); - - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; j += 16) { - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15 16 17 - // 18 19 20 21 22 23 - const __m256i data = _mm256_inserti128_si256( - _mm256_loadu_si256((__m256i *)&src_ptr[(i * src_stride) + j]), - _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + (j + 8)]), - 1); - - __m256i res_16b = - convolve_lowbd_x_4tap(data, coeffs_256 + 1, filt_256); - res_16b = sr_x_round_avx2(res_16b); - - /* rounding code */ - // 8 bit conversion and saturation to uint8 - __m256i res_8b = _mm256_packus_epi16(res_16b, res_16b); - - // Store values into the destination buffer - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - res_8b = _mm256_permute4x64_epi64(res_8b, 216); - __m128i res = _mm256_castsi256_si128(res_8b); - _mm_storeu_si128((__m128i *)&dst[i * dst_stride + j], res); - } - } - } - } else { - __m256i filt_256[4]; - - filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx2); - filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx2); - filt_256[2] = _mm256_loadu_si256((__m256i const *)filt3_global_avx2); - - if (horz_tap == 6) { - // horz_filt as 6 tap - const uint8_t *src_ptr = src - 2; - - prepare_half_coeffs_6tap_avx2(filter_params_x, subpel_x_q4, coeffs_256); - - if (w == 8) { - do { - const __m256i res = x_convolve_6tap_8x2_avx2(src_ptr, src_stride, - coeffs_256, filt_256); - sr_x_round_store_8x2_avx2(res, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - do { - __m256i r[2]; - - x_convolve_6tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, - r); - sr_x_round_store_16x2_avx2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - do { - sr_x_6tap_32_avx2(src_ptr, coeffs_256, filt_256, dst); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else if (w == 64) { - do { - sr_x_6tap_32_avx2(src_ptr, coeffs_256, filt_256, dst); - sr_x_6tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, dst + 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else { - assert(w == 128); - - do { - sr_x_6tap_32_avx2(src_ptr, coeffs_256, filt_256, dst); - sr_x_6tap_32_avx2(src_ptr + 1 * 32, coeffs_256, filt_256, - dst + 1 * 32); - sr_x_6tap_32_avx2(src_ptr + 2 * 32, coeffs_256, filt_256, - dst + 2 * 32); - sr_x_6tap_32_avx2(src_ptr + 3 * 32, coeffs_256, filt_256, - dst + 3 * 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } - } else if (horz_tap == 8) { - // horz_filt as 8 tap - const uint8_t *src_ptr = src - 3; - - filt_256[3] = _mm256_loadu_si256((__m256i const *)filt4_global_avx2); - - prepare_half_coeffs_8tap_avx2(filter_params_x, subpel_x_q4, coeffs_256); - - if (w == 8) { - do { - const __m256i res = x_convolve_8tap_8x2_avx2(src_ptr, src_stride, - coeffs_256, filt_256); - sr_x_round_store_8x2_avx2(res, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 16) { - do { - __m256i r[2]; - - x_convolve_8tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, - r); - sr_x_round_store_16x2_avx2(r, dst, dst_stride); - src_ptr += 2 * src_stride; - dst += 2 * dst_stride; - y -= 2; - } while (y); - } else if (w == 32) { - do { - sr_x_8tap_32_avx2(src_ptr, coeffs_256, filt_256, dst); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else if (w == 64) { - do { - sr_x_8tap_32_avx2(src_ptr, coeffs_256, filt_256, dst); - sr_x_8tap_32_avx2(src_ptr + 32, coeffs_256, filt_256, dst + 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } else { - assert(w == 128); - - do { - sr_x_8tap_32_avx2(src_ptr, coeffs_256, filt_256, dst); - sr_x_8tap_32_avx2(src_ptr + 1 * 32, coeffs_256, filt_256, - dst + 1 * 32); - sr_x_8tap_32_avx2(src_ptr + 2 * 32, coeffs_256, filt_256, - dst + 2 * 32); - sr_x_8tap_32_avx2(src_ptr + 3 * 32, coeffs_256, filt_256, - dst + 3 * 32); - src_ptr += src_stride; - dst += dst_stride; - } while (--y); - } - } - } -} - -#endif // THIRD_PARTY_SVT_AV1_CONVOLVE_AVX2_H_
diff --git a/third_party/SVT-AV1/synonyms.h b/third_party/SVT-AV1/synonyms.h deleted file mode 100644 index 736be4d..0000000 --- a/third_party/SVT-AV1/synonyms.h +++ /dev/null
@@ -1,31 +0,0 @@ -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef AOM_THIRD_PARTY_SVT_AV1_SYNONYMS_H_ -#define AOM_THIRD_PARTY_SVT_AV1_SYNONYMS_H_ - -#include "aom_dsp/x86/mem_sse2.h" -#include "aom_dsp/x86/synonyms.h" - -static inline __m128i load_u8_8x2_sse2(const uint8_t *const src, - const ptrdiff_t stride) { - return load_8bit_8x2_to_1_reg_sse2(src, (int)(sizeof(*src) * stride)); -} - -static AOM_FORCE_INLINE void store_u8_4x2_sse2(const __m128i src, - uint8_t *const dst, - const ptrdiff_t stride) { - xx_storel_32(dst, src); - *(uint32_t *)(dst + stride) = - ((uint32_t)_mm_extract_epi16(src, 3) << 16) | _mm_extract_epi16(src, 2); -} - -#endif // AOM_THIRD_PARTY_SVT_AV1_SYNONYMS_H_