| /* | |
| * Copyright (c) 2018, Alliance for Open Media. All rights reserved | |
| * | |
| * This source code is subject to the terms of the BSD 2 Clause License and | |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License | |
| * was not distributed with this source code in the LICENSE file, you can | |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open | |
| * Media Patent License 1.0 was not distributed with this source code in the | |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. | |
| */ | |
| #ifndef AOM_AOM_DSP_X86_TRANSPOSE_SSE2_H_ | |
| #define AOM_AOM_DSP_X86_TRANSPOSE_SSE2_H_ | |
| #include <emmintrin.h> // SSE2 | |
| #include "config/aom_config.h" | |
| static INLINE __m128i transpose_8bit_4x4(const __m128i *const in) { | |
| // Unpack 8 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 | |
| // in[1]: 10 11 12 13 | |
| // in[2]: 20 21 22 23 | |
| // in[3]: 30 31 32 33 | |
| // to: | |
| // a0: 00 10 01 11 02 12 03 13 | |
| // a1: 20 30 21 31 22 32 23 33 | |
| const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]); | |
| // Unpack 16 bit elements resulting in: | |
| // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 | |
| return _mm_unpacklo_epi16(a0, a1); | |
| } | |
| static INLINE void transpose_8bit_8x8(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 8 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 04 05 06 07 | |
| // in[1]: 10 11 12 13 14 15 16 17 | |
| // in[2]: 20 21 22 23 24 25 26 27 | |
| // in[3]: 30 31 32 33 34 35 36 37 | |
| // in[4]: 40 41 42 43 44 45 46 47 | |
| // in[5]: 50 51 52 53 54 55 56 57 | |
| // in[6]: 60 61 62 63 64 65 66 67 | |
| // in[7]: 70 71 72 73 74 75 76 77 | |
| // to: | |
| // a0: 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17 | |
| // a1: 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37 | |
| // a2: 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57 | |
| // a3: 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77 | |
| const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]); | |
| const __m128i a2 = _mm_unpacklo_epi8(in[4], in[5]); | |
| const __m128i a3 = _mm_unpacklo_epi8(in[6], in[7]); | |
| // Unpack 16 bit elements resulting in: | |
| // b0: 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 | |
| // b1: 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73 | |
| // b2: 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 | |
| // b3: 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77 | |
| const __m128i b0 = _mm_unpacklo_epi16(a0, a1); | |
| const __m128i b1 = _mm_unpackhi_epi16(a0, a1); | |
| const __m128i b2 = _mm_unpacklo_epi16(a2, a3); | |
| const __m128i b3 = _mm_unpackhi_epi16(a2, a3); | |
| // Unpack 32 bit elements resulting in: | |
| // c0: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71 | |
| // c1: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73 | |
| // c2: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75 | |
| // c3: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77 | |
| const __m128i c0 = _mm_unpacklo_epi32(b0, b2); | |
| const __m128i c1 = _mm_unpackhi_epi32(b0, b2); | |
| const __m128i c2 = _mm_unpacklo_epi32(b1, b3); | |
| const __m128i c3 = _mm_unpackhi_epi32(b1, b3); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 40 50 60 70 | |
| // out[1]: 01 11 21 31 41 51 61 71 | |
| // out[2]: 02 12 22 32 42 52 62 72 | |
| // out[3]: 03 13 23 33 43 53 63 73 | |
| // out[4]: 04 14 24 34 44 54 64 74 | |
| // out[5]: 05 15 25 35 45 55 65 75 | |
| // out[6]: 06 16 26 36 46 56 66 76 | |
| // out[7]: 07 17 27 37 47 57 67 77 | |
| out[0] = _mm_unpacklo_epi64(c0, c0); | |
| out[1] = _mm_unpackhi_epi64(c0, c0); | |
| out[2] = _mm_unpacklo_epi64(c1, c1); | |
| out[3] = _mm_unpackhi_epi64(c1, c1); | |
| out[4] = _mm_unpacklo_epi64(c2, c2); | |
| out[5] = _mm_unpackhi_epi64(c2, c2); | |
| out[6] = _mm_unpacklo_epi64(c3, c3); | |
| out[7] = _mm_unpackhi_epi64(c3, c3); | |
| } | |
| static INLINE void transpose_16bit_4x4(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 16 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 XX XX XX XX | |
| // in[1]: 10 11 12 13 XX XX XX XX | |
| // in[2]: 20 21 22 23 XX XX XX XX | |
| // in[3]: 30 31 32 33 XX XX XX XX | |
| // to: | |
| // a0: 00 10 01 11 02 12 03 13 | |
| // a1: 20 30 21 31 22 32 23 33 | |
| const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]); | |
| // Unpack 32 bit elements resulting in: | |
| // out[0]: 00 10 20 30 01 11 21 31 | |
| // out[1]: 01 11 21 31 __ __ __ __ | |
| // out[2]: 02 12 22 32 03 13 23 33 | |
| // out[3]: 03 13 23 33 __ __ __ __ | |
| // | |
| // Note: The high 64 bits of the output registers are shown for informational | |
| // purposes only. Callers should only use the low 64 bits of the output | |
| // registers. "__" indicates zeros. | |
| out[0] = _mm_unpacklo_epi32(a0, a1); | |
| out[1] = _mm_srli_si128(out[0], 8); | |
| out[2] = _mm_unpackhi_epi32(a0, a1); | |
| out[3] = _mm_srli_si128(out[2], 8); | |
| } | |
| static INLINE void transpose_16bit_4x8(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 16 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 XX XX XX XX | |
| // in[1]: 10 11 12 13 XX XX XX XX | |
| // in[2]: 20 21 22 23 XX XX XX XX | |
| // in[3]: 30 31 32 33 XX XX XX XX | |
| // in[4]: 40 41 42 43 XX XX XX XX | |
| // in[5]: 50 51 52 53 XX XX XX XX | |
| // in[6]: 60 61 62 63 XX XX XX XX | |
| // in[7]: 70 71 72 73 XX XX XX XX | |
| // to: | |
| // a0: 00 10 01 11 02 12 03 13 | |
| // a1: 20 30 21 31 22 32 23 33 | |
| // a2: 40 50 41 51 42 52 43 53 | |
| // a3: 60 70 61 71 62 72 63 73 | |
| const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]); | |
| const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]); | |
| const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]); | |
| // Unpack 32 bit elements resulting in: | |
| // b0: 00 10 20 30 01 11 21 31 | |
| // b1: 40 50 60 70 41 51 61 71 | |
| // b2: 02 12 22 32 03 13 23 33 | |
| // b3: 42 52 62 72 43 53 63 73 | |
| const __m128i b0 = _mm_unpacklo_epi32(a0, a1); | |
| const __m128i b1 = _mm_unpacklo_epi32(a2, a3); | |
| const __m128i b2 = _mm_unpackhi_epi32(a0, a1); | |
| const __m128i b3 = _mm_unpackhi_epi32(a2, a3); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 40 50 60 70 | |
| // out[1]: 01 11 21 31 41 51 61 71 | |
| // out[2]: 02 12 22 32 42 52 62 72 | |
| // out[3]: 03 13 23 33 43 53 63 73 | |
| out[0] = _mm_unpacklo_epi64(b0, b1); | |
| out[1] = _mm_unpackhi_epi64(b0, b1); | |
| out[2] = _mm_unpacklo_epi64(b2, b3); | |
| out[3] = _mm_unpackhi_epi64(b2, b3); | |
| } | |
| static INLINE void transpose_16bit_8x4(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 16 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 04 05 06 07 | |
| // in[1]: 10 11 12 13 14 15 16 17 | |
| // in[2]: 20 21 22 23 24 25 26 27 | |
| // in[3]: 30 31 32 33 34 35 36 37 | |
| // to: | |
| // a0: 00 10 01 11 02 12 03 13 | |
| // a1: 20 30 21 31 22 32 23 33 | |
| // a4: 04 14 05 15 06 16 07 17 | |
| // a5: 24 34 25 35 26 36 27 37 | |
| const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]); | |
| const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]); | |
| const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]); | |
| // Unpack 32 bit elements resulting in: | |
| // b0: 00 10 20 30 01 11 21 31 | |
| // b2: 04 14 24 34 05 15 25 35 | |
| // b4: 02 12 22 32 03 13 23 33 | |
| // b6: 06 16 26 36 07 17 27 37 | |
| const __m128i b0 = _mm_unpacklo_epi32(a0, a1); | |
| const __m128i b2 = _mm_unpacklo_epi32(a4, a5); | |
| const __m128i b4 = _mm_unpackhi_epi32(a0, a1); | |
| const __m128i b6 = _mm_unpackhi_epi32(a4, a5); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 XX XX XX XX | |
| // out[1]: 01 11 21 31 XX XX XX XX | |
| // out[2]: 02 12 22 32 XX XX XX XX | |
| // out[3]: 03 13 23 33 XX XX XX XX | |
| // out[4]: 04 14 24 34 XX XX XX XX | |
| // out[5]: 05 15 25 35 XX XX XX XX | |
| // out[6]: 06 16 26 36 XX XX XX XX | |
| // out[7]: 07 17 27 37 XX XX XX XX | |
| const __m128i zeros = _mm_setzero_si128(); | |
| out[0] = _mm_unpacklo_epi64(b0, zeros); | |
| out[1] = _mm_unpackhi_epi64(b0, zeros); | |
| out[2] = _mm_unpacklo_epi64(b4, zeros); | |
| out[3] = _mm_unpackhi_epi64(b4, zeros); | |
| out[4] = _mm_unpacklo_epi64(b2, zeros); | |
| out[5] = _mm_unpackhi_epi64(b2, zeros); | |
| out[6] = _mm_unpacklo_epi64(b6, zeros); | |
| out[7] = _mm_unpackhi_epi64(b6, zeros); | |
| } | |
| static INLINE void transpose_16bit_8x8(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 16 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 04 05 06 07 | |
| // in[1]: 10 11 12 13 14 15 16 17 | |
| // in[2]: 20 21 22 23 24 25 26 27 | |
| // in[3]: 30 31 32 33 34 35 36 37 | |
| // in[4]: 40 41 42 43 44 45 46 47 | |
| // in[5]: 50 51 52 53 54 55 56 57 | |
| // in[6]: 60 61 62 63 64 65 66 67 | |
| // in[7]: 70 71 72 73 74 75 76 77 | |
| // to: | |
| // a0: 00 10 01 11 02 12 03 13 | |
| // a1: 20 30 21 31 22 32 23 33 | |
| // a2: 40 50 41 51 42 52 43 53 | |
| // a3: 60 70 61 71 62 72 63 73 | |
| // a4: 04 14 05 15 06 16 07 17 | |
| // a5: 24 34 25 35 26 36 27 37 | |
| // a6: 44 54 45 55 46 56 47 57 | |
| // a7: 64 74 65 75 66 76 67 77 | |
| const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]); | |
| const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]); | |
| const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]); | |
| const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]); | |
| const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]); | |
| const __m128i a6 = _mm_unpackhi_epi16(in[4], in[5]); | |
| const __m128i a7 = _mm_unpackhi_epi16(in[6], in[7]); | |
| // Unpack 32 bit elements resulting in: | |
| // b0: 00 10 20 30 01 11 21 31 | |
| // b1: 40 50 60 70 41 51 61 71 | |
| // b2: 04 14 24 34 05 15 25 35 | |
| // b3: 44 54 64 74 45 55 65 75 | |
| // b4: 02 12 22 32 03 13 23 33 | |
| // b5: 42 52 62 72 43 53 63 73 | |
| // b6: 06 16 26 36 07 17 27 37 | |
| // b7: 46 56 66 76 47 57 67 77 | |
| const __m128i b0 = _mm_unpacklo_epi32(a0, a1); | |
| const __m128i b1 = _mm_unpacklo_epi32(a2, a3); | |
| const __m128i b2 = _mm_unpacklo_epi32(a4, a5); | |
| const __m128i b3 = _mm_unpacklo_epi32(a6, a7); | |
| const __m128i b4 = _mm_unpackhi_epi32(a0, a1); | |
| const __m128i b5 = _mm_unpackhi_epi32(a2, a3); | |
| const __m128i b6 = _mm_unpackhi_epi32(a4, a5); | |
| const __m128i b7 = _mm_unpackhi_epi32(a6, a7); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 40 50 60 70 | |
| // out[1]: 01 11 21 31 41 51 61 71 | |
| // out[2]: 02 12 22 32 42 52 62 72 | |
| // out[3]: 03 13 23 33 43 53 63 73 | |
| // out[4]: 04 14 24 34 44 54 64 74 | |
| // out[5]: 05 15 25 35 45 55 65 75 | |
| // out[6]: 06 16 26 36 46 56 66 76 | |
| // out[7]: 07 17 27 37 47 57 67 77 | |
| out[0] = _mm_unpacklo_epi64(b0, b1); | |
| out[1] = _mm_unpackhi_epi64(b0, b1); | |
| out[2] = _mm_unpacklo_epi64(b4, b5); | |
| out[3] = _mm_unpackhi_epi64(b4, b5); | |
| out[4] = _mm_unpacklo_epi64(b2, b3); | |
| out[5] = _mm_unpackhi_epi64(b2, b3); | |
| out[6] = _mm_unpacklo_epi64(b6, b7); | |
| out[7] = _mm_unpackhi_epi64(b6, b7); | |
| } | |
| // Transpose in-place | |
| static INLINE void transpose_16bit_16x16(__m128i *const left, | |
| __m128i *const right) { | |
| __m128i tbuf[8]; | |
| transpose_16bit_8x8(left, left); | |
| transpose_16bit_8x8(right, tbuf); | |
| transpose_16bit_8x8(left + 8, right); | |
| transpose_16bit_8x8(right + 8, right + 8); | |
| left[8] = tbuf[0]; | |
| left[9] = tbuf[1]; | |
| left[10] = tbuf[2]; | |
| left[11] = tbuf[3]; | |
| left[12] = tbuf[4]; | |
| left[13] = tbuf[5]; | |
| left[14] = tbuf[6]; | |
| left[15] = tbuf[7]; | |
| } | |
| static INLINE void transpose_32bit_4x4(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 32 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 | |
| // in[1]: 10 11 12 13 | |
| // in[2]: 20 21 22 23 | |
| // in[3]: 30 31 32 33 | |
| // to: | |
| // a0: 00 10 01 11 | |
| // a1: 20 30 21 31 | |
| // a2: 02 12 03 13 | |
| // a3: 22 32 23 33 | |
| const __m128i a0 = _mm_unpacklo_epi32(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi32(in[2], in[3]); | |
| const __m128i a2 = _mm_unpackhi_epi32(in[0], in[1]); | |
| const __m128i a3 = _mm_unpackhi_epi32(in[2], in[3]); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 | |
| // out[1]: 01 11 21 31 | |
| // out[2]: 02 12 22 32 | |
| // out[3]: 03 13 23 33 | |
| out[0] = _mm_unpacklo_epi64(a0, a1); | |
| out[1] = _mm_unpackhi_epi64(a0, a1); | |
| out[2] = _mm_unpacklo_epi64(a2, a3); | |
| out[3] = _mm_unpackhi_epi64(a2, a3); | |
| } | |
| static INLINE void transpose_32bit_4x4x2(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 32 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 | |
| // in[1]: 10 11 12 13 | |
| // in[2]: 20 21 22 23 | |
| // in[3]: 30 31 32 33 | |
| // in[4]: 04 05 06 07 | |
| // in[5]: 14 15 16 17 | |
| // in[6]: 24 25 26 27 | |
| // in[7]: 34 35 36 37 | |
| // to: | |
| // a0: 00 10 01 11 | |
| // a1: 20 30 21 31 | |
| // a2: 02 12 03 13 | |
| // a3: 22 32 23 33 | |
| // a4: 04 14 05 15 | |
| // a5: 24 34 25 35 | |
| // a6: 06 16 07 17 | |
| // a7: 26 36 27 37 | |
| const __m128i a0 = _mm_unpacklo_epi32(in[0], in[1]); | |
| const __m128i a1 = _mm_unpacklo_epi32(in[2], in[3]); | |
| const __m128i a2 = _mm_unpackhi_epi32(in[0], in[1]); | |
| const __m128i a3 = _mm_unpackhi_epi32(in[2], in[3]); | |
| const __m128i a4 = _mm_unpacklo_epi32(in[4], in[5]); | |
| const __m128i a5 = _mm_unpacklo_epi32(in[6], in[7]); | |
| const __m128i a6 = _mm_unpackhi_epi32(in[4], in[5]); | |
| const __m128i a7 = _mm_unpackhi_epi32(in[6], in[7]); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 | |
| // out[1]: 01 11 21 31 | |
| // out[2]: 02 12 22 32 | |
| // out[3]: 03 13 23 33 | |
| // out[4]: 04 14 24 34 | |
| // out[5]: 05 15 25 35 | |
| // out[6]: 06 16 26 36 | |
| // out[7]: 07 17 27 37 | |
| out[0] = _mm_unpacklo_epi64(a0, a1); | |
| out[1] = _mm_unpackhi_epi64(a0, a1); | |
| out[2] = _mm_unpacklo_epi64(a2, a3); | |
| out[3] = _mm_unpackhi_epi64(a2, a3); | |
| out[4] = _mm_unpacklo_epi64(a4, a5); | |
| out[5] = _mm_unpackhi_epi64(a4, a5); | |
| out[6] = _mm_unpacklo_epi64(a6, a7); | |
| out[7] = _mm_unpackhi_epi64(a6, a7); | |
| } | |
| static INLINE void transpose_32bit_8x4(const __m128i *const in, | |
| __m128i *const out) { | |
| // Unpack 32 bit elements. Goes from: | |
| // in[0]: 00 01 02 03 | |
| // in[1]: 04 05 06 07 | |
| // in[2]: 10 11 12 13 | |
| // in[3]: 14 15 16 17 | |
| // in[4]: 20 21 22 23 | |
| // in[5]: 24 25 26 27 | |
| // in[6]: 30 31 32 33 | |
| // in[7]: 34 35 36 37 | |
| // to: | |
| // a0: 00 10 01 11 | |
| // a1: 20 30 21 31 | |
| // a2: 02 12 03 13 | |
| // a3: 22 32 23 33 | |
| // a4: 04 14 05 15 | |
| // a5: 24 34 25 35 | |
| // a6: 06 16 07 17 | |
| // a7: 26 36 27 37 | |
| const __m128i a0 = _mm_unpacklo_epi32(in[0], in[2]); | |
| const __m128i a1 = _mm_unpacklo_epi32(in[4], in[6]); | |
| const __m128i a2 = _mm_unpackhi_epi32(in[0], in[2]); | |
| const __m128i a3 = _mm_unpackhi_epi32(in[4], in[6]); | |
| const __m128i a4 = _mm_unpacklo_epi32(in[1], in[3]); | |
| const __m128i a5 = _mm_unpacklo_epi32(in[5], in[7]); | |
| const __m128i a6 = _mm_unpackhi_epi32(in[1], in[3]); | |
| const __m128i a7 = _mm_unpackhi_epi32(in[5], in[7]); | |
| // Unpack 64 bit elements resulting in: | |
| // out[0]: 00 10 20 30 | |
| // out[1]: 01 11 21 31 | |
| // out[2]: 02 12 22 32 | |
| // out[3]: 03 13 23 33 | |
| // out[4]: 04 14 24 34 | |
| // out[5]: 05 15 25 35 | |
| // out[6]: 06 16 26 36 | |
| // out[7]: 07 17 27 37 | |
| out[0] = _mm_unpacklo_epi64(a0, a1); | |
| out[1] = _mm_unpackhi_epi64(a0, a1); | |
| out[2] = _mm_unpacklo_epi64(a2, a3); | |
| out[3] = _mm_unpackhi_epi64(a2, a3); | |
| out[4] = _mm_unpacklo_epi64(a4, a5); | |
| out[5] = _mm_unpackhi_epi64(a4, a5); | |
| out[6] = _mm_unpacklo_epi64(a6, a7); | |
| out[7] = _mm_unpackhi_epi64(a6, a7); | |
| } | |
| #endif // AOM_AOM_DSP_X86_TRANSPOSE_SSE2_H_ |