| /* |
| * Copyright (c) 2018, Alliance for Open Media. All rights reserved. |
| * |
| * This source code is subject to the terms of the BSD 2 Clause License and |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| * was not distributed with this source code in the LICENSE file, you can |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| * Media Patent License 1.0 was not distributed with this source code in the |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| */ |
| |
| #include <assert.h> |
| #include <smmintrin.h> |
| #include <string.h> |
| |
| #include "config/av1_rtcd.h" |
| |
| #include "aom_dsp/x86/synonyms.h" |
| #include "av1/common/enums.h" |
| #include "av1/common/reconintra.h" |
| |
| //------------------------------------------------------------------------------ |
| // filter_intra_predictor_sse4_1 |
| |
| // This shuffle mask selects 32-bit blocks in the order 0, 1, 0, 1, which |
| // duplicates the first 8 bytes of a 128-bit vector into the second 8 bytes. |
| #define DUPLICATE_FIRST_HALF 0x44 |
| |
| // Apply all filter taps to the given 7 packed 16-bit values, keeping the 8th |
| // at zero to preserve the sum. |
| static inline void filter_4x2_sse4_1(uint8_t *dst, const ptrdiff_t stride, |
| const __m128i *pixels, |
| const __m128i *taps_0_1, |
| const __m128i *taps_2_3, |
| const __m128i *taps_4_5, |
| const __m128i *taps_6_7) { |
| const __m128i mul_0_01 = _mm_maddubs_epi16(*pixels, *taps_0_1); |
| const __m128i mul_0_23 = _mm_maddubs_epi16(*pixels, *taps_2_3); |
| // |output_half| contains 8 partial sums. |
| __m128i output_half = _mm_hadd_epi16(mul_0_01, mul_0_23); |
| __m128i output = _mm_hadd_epi16(output_half, output_half); |
| const __m128i output_row0 = |
| _mm_packus_epi16(xx_roundn_epi16_unsigned(output, 4), |
| /* arbitrary pack arg */ output); |
| xx_storel_32(dst, output_row0); |
| const __m128i mul_1_01 = _mm_maddubs_epi16(*pixels, *taps_4_5); |
| const __m128i mul_1_23 = _mm_maddubs_epi16(*pixels, *taps_6_7); |
| output_half = _mm_hadd_epi16(mul_1_01, mul_1_23); |
| output = _mm_hadd_epi16(output_half, output_half); |
| const __m128i output_row1 = |
| _mm_packus_epi16(xx_roundn_epi16_unsigned(output, 4), |
| /* arbitrary pack arg */ output); |
| xx_storel_32(dst + stride, output_row1); |
| } |
| |
| // 4xH transform sizes are given special treatment because xx_loadl_64 goes out |
| // of bounds and every block involves the left column. This implementation |
| // loads TL from the top row for the first block, so it is not |
| static inline void filter_4xh(uint8_t *dest, ptrdiff_t stride, |
| const uint8_t *const top_ptr, |
| const uint8_t *const left_ptr, int mode, |
| const int height) { |
| const __m128i taps_0_1 = xx_load_128(av1_filter_intra_taps[mode][0]); |
| const __m128i taps_2_3 = xx_load_128(av1_filter_intra_taps[mode][2]); |
| const __m128i taps_4_5 = xx_load_128(av1_filter_intra_taps[mode][4]); |
| const __m128i taps_6_7 = xx_load_128(av1_filter_intra_taps[mode][6]); |
| __m128i top = xx_loadl_32(top_ptr - 1); |
| __m128i pixels = _mm_insert_epi8(top, (int8_t)top_ptr[3], 4); |
| __m128i left = (height == 4 ? xx_loadl_32(left_ptr) : xx_loadl_64(left_ptr)); |
| left = _mm_slli_si128(left, 5); |
| |
| // Relative pixels: top[-1], top[0], top[1], top[2], top[3], left[0], left[1], |
| // left[2], left[3], left[4], left[5], left[6], left[7] |
| pixels = _mm_or_si128(left, pixels); |
| |
| // Duplicate first 8 bytes. |
| pixels = _mm_shuffle_epi32(pixels, DUPLICATE_FIRST_HALF); |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| dest += stride; // Move to y = 1. |
| pixels = xx_loadl_32(dest); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, left[-2], left[-1], |
| // left[0], left[1], ... |
| pixels = _mm_or_si128(left, pixels); |
| |
| // This mask rearranges bytes in the order: 6, 0, 1, 2, 3, 7, 8, 15. The last |
| // byte is an unused value, which shall be multiplied by 0 when we apply the |
| // filter. |
| const int64_t kInsertTopLeftFirstMask = 0x0F08070302010006; |
| |
| // Insert left[-1] in front as TL and put left[0] and left[1] at the end. |
| const __m128i pixel_order1 = _mm_set1_epi64x(kInsertTopLeftFirstMask); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 2. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| dest += stride; // Move to y = 3. |
| |
| // Compute the middle 8 rows before using common code for the final 4 rows. |
| // Because the common code below this block assumes that |
| if (height == 16) { |
| // This shift allows us to use pixel_order2 twice after shifting by 2 later. |
| left = _mm_slli_si128(left, 1); |
| pixels = xx_loadl_32(dest); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, empty, left[-4], |
| // left[-3], left[-2], left[-1], left[0], left[1], left[2], left[3] |
| pixels = _mm_or_si128(left, pixels); |
| |
| // This mask rearranges bytes in the order: 9, 0, 1, 2, 3, 7, 8, 15. The |
| // last byte is an unused value, as above. The top-left was shifted to |
| // position nine to keep two empty spaces after the top pixels. |
| const int64_t kInsertTopLeftSecondMask = 0x0F0B0A0302010009; |
| |
| // Insert (relative) left[-1] in front as TL and put left[0] and left[1] at |
| // the end. |
| const __m128i pixel_order2 = _mm_set1_epi64x(kInsertTopLeftSecondMask); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| dest += stride; // Move to y = 4. |
| |
| // First 4x2 in the if body. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| |
| // Clear all but final pixel in the first 8 of left column. |
| __m128i keep_top_left = _mm_srli_si128(left, 13); |
| dest += stride; // Move to y = 5. |
| pixels = xx_loadl_32(dest); |
| left = _mm_srli_si128(left, 2); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], left[-6], |
| // left[-5], left[-4], left[-3], left[-2], left[-1], left[0], left[1] |
| pixels = _mm_or_si128(left, pixels); |
| left = xx_loadl_64(left_ptr + 8); |
| |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| dest += stride; // Move to y = 6. |
| |
| // Second 4x2 in the if body. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| |
| // Position TL value so we can use pixel_order1. |
| keep_top_left = _mm_slli_si128(keep_top_left, 6); |
| dest += stride; // Move to y = 7. |
| pixels = xx_loadl_32(dest); |
| left = _mm_slli_si128(left, 7); |
| left = _mm_or_si128(left, keep_top_left); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, empty, |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 8. |
| |
| // Third 4x2 in the if body. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| dest += stride; // Move to y = 9. |
| |
| // Prepare final inputs. |
| pixels = xx_loadl_32(dest); |
| left = _mm_srli_si128(left, 2); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], left[-3], left[-2] |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 10. |
| |
| // Fourth 4x2 in the if body. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| dest += stride; // Move to y = 11. |
| } |
| |
| // In both the 8 and 16 case, we assume that the left vector has the next TL |
| // at position 8. |
| if (height > 4) { |
| // Erase prior left pixels by shifting TL to position 0. |
| left = _mm_srli_si128(left, 8); |
| left = _mm_slli_si128(left, 6); |
| pixels = xx_loadl_32(dest); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, empty, |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 12 or 4. |
| |
| // First of final two 4x2 blocks. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| dest += stride; // Move to y = 13 or 5. |
| pixels = xx_loadl_32(dest); |
| left = _mm_srli_si128(left, 2); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], left[-3], left[-2] |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 14 or 6. |
| |
| // Last of final two 4x2 blocks. |
| filter_4x2_sse4_1(dest, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| } |
| } |
| |
| static inline void filter_intra_predictor_sse4_1(void *const dest, |
| ptrdiff_t stride, |
| const void *const top_row, |
| const void *const left_column, |
| int mode, const int width, |
| const int height) { |
| const uint8_t *const top_ptr = (const uint8_t *)top_row; |
| const uint8_t *const left_ptr = (const uint8_t *)left_column; |
| uint8_t *dst = (uint8_t *)dest; |
| if (width == 4) { |
| filter_4xh(dst, stride, top_ptr, left_ptr, mode, height); |
| return; |
| } |
| |
| // There is one set of 7 taps for each of the 4x2 output pixels. |
| const __m128i taps_0_1 = xx_load_128(av1_filter_intra_taps[mode][0]); |
| const __m128i taps_2_3 = xx_load_128(av1_filter_intra_taps[mode][2]); |
| const __m128i taps_4_5 = xx_load_128(av1_filter_intra_taps[mode][4]); |
| const __m128i taps_6_7 = xx_load_128(av1_filter_intra_taps[mode][6]); |
| |
| // This mask rearranges bytes in the order: 0, 1, 2, 3, 4, 8, 9, 15. The 15 at |
| // the end is an unused value, which shall be multiplied by 0 when we apply |
| // the filter. |
| const int64_t kCondenseLeftMask = 0x0F09080403020100; |
| |
| // Takes the "left section" and puts it right after p0-p4. |
| const __m128i pixel_order1 = _mm_set1_epi64x(kCondenseLeftMask); |
| |
| // This mask rearranges bytes in the order: 8, 0, 1, 2, 3, 9, 10, 15. The last |
| // byte is unused as above. |
| const int64_t kInsertTopLeftMask = 0x0F0A090302010008; |
| |
| // Shuffles the "top left" from the left section, to the front. Used when |
| // grabbing data from left_column and not top_row. |
| const __m128i pixel_order2 = _mm_set1_epi64x(kInsertTopLeftMask); |
| |
| // This first pass takes care of the cases where the top left pixel comes from |
| // top_row. |
| __m128i pixels = xx_loadl_64(top_ptr - 1); |
| __m128i left = _mm_slli_si128(xx_loadl_32(left_column), 8); |
| pixels = _mm_or_si128(pixels, left); |
| |
| // Two sets of the same pixels to multiply with two sets of taps. |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| filter_4x2_sse4_1(dst, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| left = _mm_srli_si128(left, 1); |
| |
| // Load |
| pixels = xx_loadl_32(dst + stride); |
| |
| // Because of the above shift, this OR 'invades' the final of the first 8 |
| // bytes of |pixels|. This is acceptable because the 8th filter tap is always |
| // a padded 0. |
| pixels = _mm_or_si128(pixels, left); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| const ptrdiff_t stride2 = stride << 1; |
| const ptrdiff_t stride4 = stride << 2; |
| filter_4x2_sse4_1(dst + stride2, stride, &pixels, &taps_0_1, &taps_2_3, |
| &taps_4_5, &taps_6_7); |
| dst += 4; |
| for (int x = 3; x < width - 4; x += 4) { |
| pixels = xx_loadl_32(top_ptr + x); |
| pixels = _mm_insert_epi8(pixels, (int8_t)top_ptr[x + 4], 4); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[-1], 5); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, DUPLICATE_FIRST_HALF); |
| filter_4x2_sse4_1(dst, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| pixels = xx_loadl_32(dst + stride - 1); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride + 3], 4); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride2 - 1], 5); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride + stride2 - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, DUPLICATE_FIRST_HALF); |
| filter_4x2_sse4_1(dst + stride2, stride, &pixels, &taps_0_1, &taps_2_3, |
| &taps_4_5, &taps_6_7); |
| dst += 4; |
| } |
| |
| // Now we handle heights that reference previous blocks rather than top_row. |
| for (int y = 4; y < height; y += 4) { |
| // Leftmost 4x4 block for this height. |
| dst -= width; |
| dst += stride4; |
| |
| // Top Left is not available by offset in these leftmost blocks. |
| pixels = xx_loadl_32(dst - stride); |
| left = _mm_slli_si128(xx_loadl_32(left_ptr + y - 1), 8); |
| left = _mm_insert_epi8(left, (int8_t)left_ptr[y + 3], 12); |
| pixels = _mm_or_si128(pixels, left); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| filter_4x2_sse4_1(dst, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| |
| // The bytes shifted into positions 6 and 7 will be ignored by the shuffle. |
| left = _mm_srli_si128(left, 2); |
| pixels = xx_loadl_32(dst + stride); |
| pixels = _mm_or_si128(pixels, left); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| filter_4x2_sse4_1(dst + stride2, stride, &pixels, &taps_0_1, &taps_2_3, |
| &taps_4_5, &taps_6_7); |
| |
| dst += 4; |
| |
| // Remaining 4x4 blocks for this height. |
| for (int x = 4; x < width; x += 4) { |
| pixels = xx_loadl_32(dst - stride - 1); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[-stride + 3], 4); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[-1], 5); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, DUPLICATE_FIRST_HALF); |
| filter_4x2_sse4_1(dst, stride, &pixels, &taps_0_1, &taps_2_3, &taps_4_5, |
| &taps_6_7); |
| pixels = xx_loadl_32(dst + stride - 1); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride + 3], 4); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride2 - 1], 5); |
| pixels = _mm_insert_epi8(pixels, (int8_t)dst[stride2 + stride - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, DUPLICATE_FIRST_HALF); |
| filter_4x2_sse4_1(dst + stride2, stride, &pixels, &taps_0_1, &taps_2_3, |
| &taps_4_5, &taps_6_7); |
| dst += 4; |
| } |
| } |
| } |
| |
| void av1_filter_intra_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, |
| TX_SIZE tx_size, const uint8_t *above, |
| const uint8_t *left, int mode) { |
| const int bw = tx_size_wide[tx_size]; |
| const int bh = tx_size_high[tx_size]; |
| filter_intra_predictor_sse4_1(dst, stride, above, left, mode, bw, bh); |
| } |