| /* | |
| * Copyright (c) 2016, Alliance for Open Media. All rights reserved | |
| * | |
| * This source code is subject to the terms of the BSD 2 Clause License and | |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License | |
| * was not distributed with this source code in the LICENSE file, you can | |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open | |
| * Media Patent License 1.0 was not distributed with this source code in the | |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. | |
| */ | |
| #include <arm_neon.h> | |
| #include <assert.h> | |
| #include <math.h> | |
| #include "av1/common/txb_common.h" | |
| #include "av1/encoder/encodetxb.h" | |
| #include "av1/common/arm/mem_neon.h" | |
| void av1_txb_init_levels_neon(const tran_low_t *const coeff, const int width, | |
| const int height, uint8_t *const levels) { | |
| const int stride = width + TX_PAD_HOR; | |
| memset(levels - TX_PAD_TOP * stride, 0, | |
| sizeof(*levels) * TX_PAD_TOP * stride); | |
| memset(levels + stride * height, 0, | |
| sizeof(*levels) * (TX_PAD_BOTTOM * stride + TX_PAD_END)); | |
| const int32x4_t zeros = vdupq_n_s32(0); | |
| int i = 0; | |
| uint8_t *ls = levels; | |
| const tran_low_t *cf = coeff; | |
| if (width == 4) { | |
| do { | |
| const int32x4_t coeffA = vld1q_s32(cf); | |
| const int32x4_t coeffB = vld1q_s32(cf + width); | |
| const int16x8_t coeffAB = | |
| vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB)); | |
| const int16x8_t absAB = vqabsq_s16(coeffAB); | |
| const int8x8_t absABs = vqmovn_s16(absAB); | |
| #if defined(__aarch64__) | |
| const int8x16_t absAB8 = | |
| vcombine_s8(absABs, vreinterpret_s8_s32(vget_low_s32(zeros))); | |
| const uint8x16_t lsAB = | |
| vreinterpretq_u8_s32(vzip1q_s32(vreinterpretq_s32_s8(absAB8), zeros)); | |
| #else | |
| const int32x2x2_t absAB8 = | |
| vzip_s32(vreinterpret_s32_s8(absABs), vget_low_s32(zeros)); | |
| const uint8x16_t lsAB = | |
| vreinterpretq_u8_s32(vcombine_s32(absAB8.val[0], absAB8.val[1])); | |
| #endif | |
| vst1q_u8(ls, lsAB); | |
| ls += (stride << 1); | |
| cf += (width << 1); | |
| i += 2; | |
| } while (i < height); | |
| } else if (width == 8) { | |
| do { | |
| const int32x4_t coeffA = vld1q_s32(cf); | |
| const int32x4_t coeffB = vld1q_s32(cf + 4); | |
| const int16x8_t coeffAB = | |
| vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB)); | |
| const int16x8_t absAB = vqabsq_s16(coeffAB); | |
| const uint8x16_t absAB8 = vreinterpretq_u8_s8(vcombine_s8( | |
| vqmovn_s16(absAB), vreinterpret_s8_s32(vget_low_s32(zeros)))); | |
| vst1q_u8(ls, absAB8); | |
| ls += stride; | |
| cf += width; | |
| i += 1; | |
| } while (i < height); | |
| } else { | |
| do { | |
| int j = 0; | |
| do { | |
| const int32x4_t coeffA = vld1q_s32(cf); | |
| const int32x4_t coeffB = vld1q_s32(cf + 4); | |
| const int32x4_t coeffC = vld1q_s32(cf + 8); | |
| const int32x4_t coeffD = vld1q_s32(cf + 12); | |
| const int16x8_t coeffAB = | |
| vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB)); | |
| const int16x8_t coeffCD = | |
| vcombine_s16(vqmovn_s32(coeffC), vqmovn_s32(coeffD)); | |
| const int16x8_t absAB = vqabsq_s16(coeffAB); | |
| const int16x8_t absCD = vqabsq_s16(coeffCD); | |
| const uint8x16_t absABCD = vreinterpretq_u8_s8( | |
| vcombine_s8(vqmovn_s16(absAB), vqmovn_s16(absCD))); | |
| vst1q_u8((ls + j), absABCD); | |
| j += 16; | |
| cf += 16; | |
| } while (j < width); | |
| *(int32_t *)(ls + width) = 0; | |
| ls += stride; | |
| i += 1; | |
| } while (i < height); | |
| } | |
| } | |
| // get_4_nz_map_contexts_2d coefficients: | |
| static const DECLARE_ALIGNED(16, uint8_t, c_4_po_2d[2][16]) = { | |
| { 0, 1, 6, 6, 1, 6, 6, 21, 6, 6, 21, 21, 6, 21, 21, 21 }, | |
| { 0, 11, 11, 11, 11, 11, 11, 11, 6, 6, 21, 21, 6, 21, 21, 21 } | |
| }; | |
| // get_4_nz_map_contexts_hor coefficients: | |
| /* clang-format off */ | |
| #define SIG_COEF_CONTEXTS_2D_X4_051010 \ | |
| (SIG_COEF_CONTEXTS_2D + ((SIG_COEF_CONTEXTS_2D + 5) << 8) + \ | |
| ((SIG_COEF_CONTEXTS_2D + 10) << 16) + ((SIG_COEF_CONTEXTS_2D + 10) << 24)) | |
| /* clang-format on */ | |
| // get_4_nz_map_contexts_ver coefficients: | |
| static const DECLARE_ALIGNED(16, uint8_t, c_4_po_ver[16]) = { | |
| SIG_COEF_CONTEXTS_2D + 0, SIG_COEF_CONTEXTS_2D + 0, | |
| SIG_COEF_CONTEXTS_2D + 0, SIG_COEF_CONTEXTS_2D + 0, | |
| SIG_COEF_CONTEXTS_2D + 5, SIG_COEF_CONTEXTS_2D + 5, | |
| SIG_COEF_CONTEXTS_2D + 5, SIG_COEF_CONTEXTS_2D + 5, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10 | |
| }; | |
| // get_8_coeff_contexts_2d coefficients: | |
| // if (height == 8) | |
| static const DECLARE_ALIGNED(16, uint8_t, c_8_po_2d_8[2][16]) = { | |
| { 0, 1, 6, 6, 21, 21, 21, 21, 1, 6, 6, 21, 21, 21, 21, 21 }, | |
| { 6, 6, 21, 21, 21, 21, 21, 21, 6, 21, 21, 21, 21, 21, 21, 21 } | |
| }; | |
| // if (height < 8) | |
| static const DECLARE_ALIGNED(16, uint8_t, c_8_po_2d_l[2][16]) = { | |
| { 0, 16, 6, 6, 21, 21, 21, 21, 16, 16, 6, 21, 21, 21, 21, 21 }, | |
| { 16, 16, 21, 21, 21, 21, 21, 21, 16, 16, 21, 21, 21, 21, 21, 21 } | |
| }; | |
| // if (height > 8) | |
| static const DECLARE_ALIGNED(16, uint8_t, c_8_po_2d_g[2][16]) = { | |
| { 0, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }, | |
| { 6, 6, 21, 21, 21, 21, 21, 21, 6, 21, 21, 21, 21, 21, 21, 21 } | |
| }; | |
| // get_4_nz_map_contexts_ver coefficients: | |
| static const DECLARE_ALIGNED(16, uint8_t, c_8_po_hor[16]) = { | |
| SIG_COEF_CONTEXTS_2D + 0, SIG_COEF_CONTEXTS_2D + 5, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 0, SIG_COEF_CONTEXTS_2D + 5, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10 | |
| }; | |
| // get_16n_coeff_contexts_2d coefficients: | |
| // real_width == real_height | |
| static const DECLARE_ALIGNED(16, uint8_t, c_16_po_2d_e[4][16]) = { | |
| { 0, 1, 6, 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 }, | |
| { 1, 6, 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 }, | |
| { 6, 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 }, | |
| { 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 } | |
| }; | |
| // real_width > real_height | |
| static const DECLARE_ALIGNED(16, uint8_t, c_16_po_2d_g[3][16]) = { | |
| { 0, 16, 6, 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 }, | |
| { 16, 16, 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 }, | |
| { 16, 16, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 } | |
| }; | |
| // real_width < real_height | |
| static const DECLARE_ALIGNED(16, uint8_t, c_16_po_2d_l[3][16]) = { | |
| { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }, | |
| { 6, 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 }, | |
| { 6, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21 } | |
| }; | |
| // get_16n_coeff_contexts_hor coefficients: | |
| static const DECLARE_ALIGNED(16, uint8_t, c_16_po_hor[16]) = { | |
| SIG_COEF_CONTEXTS_2D + 0, SIG_COEF_CONTEXTS_2D + 5, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10, | |
| SIG_COEF_CONTEXTS_2D + 10, SIG_COEF_CONTEXTS_2D + 10 | |
| }; | |
| // end of coefficients declaration area | |
| static INLINE uint8x16_t load_8bit_4x4_to_1_reg(const uint8_t *const src, | |
| const int byte_stride) { | |
| #ifdef __aarch64__ | |
| uint32x4_t v_data = vld1q_u32((uint32_t *)src); | |
| v_data = vld1q_lane_u32((uint32_t *)(src + 1 * byte_stride), v_data, 1); | |
| v_data = vld1q_lane_u32((uint32_t *)(src + 2 * byte_stride), v_data, 2); | |
| v_data = vld1q_lane_u32((uint32_t *)(src + 3 * byte_stride), v_data, 3); | |
| return vreinterpretq_u8_u32(v_data); | |
| #else | |
| return load_unaligned_u8q(src, byte_stride); | |
| #endif | |
| } | |
| static INLINE uint8x16_t load_8bit_8x2_to_1_reg(const uint8_t *const src, | |
| const int byte_stride) { | |
| #ifdef __aarch64__ | |
| uint64x2_t v_data = vld1q_u64((uint64_t *)src); | |
| v_data = vld1q_lane_u64((uint64_t *)(src + 1 * byte_stride), v_data, 1); | |
| return vreinterpretq_u8_u64(v_data); | |
| #else | |
| uint8x8_t v_data_low = vld1_u8(src); | |
| uint8x8_t v_data_high = vld1_u8(src + byte_stride); | |
| return vcombine_u8(v_data_low, v_data_high); | |
| #endif | |
| } | |
| static INLINE uint8x16_t load_8bit_16x1_to_1_reg(const uint8_t *const src, | |
| const int byte_stride) { | |
| (void)byte_stride; | |
| return vld1q_u8(src); | |
| } | |
| static INLINE void load_levels_4x4x5(const uint8_t *const src, const int stride, | |
| const ptrdiff_t *const offsets, | |
| uint8x16_t *const level) { | |
| level[0] = load_8bit_4x4_to_1_reg(&src[1], stride); | |
| level[1] = load_8bit_4x4_to_1_reg(&src[stride], stride); | |
| level[2] = load_8bit_4x4_to_1_reg(&src[offsets[0]], stride); | |
| level[3] = load_8bit_4x4_to_1_reg(&src[offsets[1]], stride); | |
| level[4] = load_8bit_4x4_to_1_reg(&src[offsets[2]], stride); | |
| } | |
| static INLINE void load_levels_8x2x5(const uint8_t *const src, const int stride, | |
| const ptrdiff_t *const offsets, | |
| uint8x16_t *const level) { | |
| level[0] = load_8bit_8x2_to_1_reg(&src[1], stride); | |
| level[1] = load_8bit_8x2_to_1_reg(&src[stride], stride); | |
| level[2] = load_8bit_8x2_to_1_reg(&src[offsets[0]], stride); | |
| level[3] = load_8bit_8x2_to_1_reg(&src[offsets[1]], stride); | |
| level[4] = load_8bit_8x2_to_1_reg(&src[offsets[2]], stride); | |
| } | |
| static INLINE void load_levels_16x1x5(const uint8_t *const src, | |
| const int stride, | |
| const ptrdiff_t *const offsets, | |
| uint8x16_t *const level) { | |
| level[0] = load_8bit_16x1_to_1_reg(&src[1], stride); | |
| level[1] = load_8bit_16x1_to_1_reg(&src[stride], stride); | |
| level[2] = load_8bit_16x1_to_1_reg(&src[offsets[0]], stride); | |
| level[3] = load_8bit_16x1_to_1_reg(&src[offsets[1]], stride); | |
| level[4] = load_8bit_16x1_to_1_reg(&src[offsets[2]], stride); | |
| } | |
| static INLINE uint8x16_t get_coeff_contexts_kernel(uint8x16_t *const level) { | |
| const uint8x16_t const_3 = vdupq_n_u8(3); | |
| const uint8x16_t const_4 = vdupq_n_u8(4); | |
| uint8x16_t count; | |
| count = vminq_u8(level[0], const_3); | |
| level[1] = vminq_u8(level[1], const_3); | |
| level[2] = vminq_u8(level[2], const_3); | |
| level[3] = vminq_u8(level[3], const_3); | |
| level[4] = vminq_u8(level[4], const_3); | |
| count = vaddq_u8(count, level[1]); | |
| count = vaddq_u8(count, level[2]); | |
| count = vaddq_u8(count, level[3]); | |
| count = vaddq_u8(count, level[4]); | |
| count = vrshrq_n_u8(count, 1); | |
| count = vminq_u8(count, const_4); | |
| return count; | |
| } | |
| static INLINE void get_4_nz_map_contexts_2d(const uint8_t *levels, | |
| const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *const coeff_contexts) { | |
| const int stride = 4 + TX_PAD_HOR; | |
| const uint8x16_t pos_to_offset_large = vdupq_n_u8(21); | |
| uint8x16_t pos_to_offset = | |
| vld1q_u8((height == 4) ? c_4_po_2d[0] : c_4_po_2d[1]); | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| uint8_t *cc = coeff_contexts; | |
| assert(!(height % 4)); | |
| int row = height; | |
| do { | |
| load_levels_4x4x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset); | |
| vst1q_u8(cc, count); | |
| pos_to_offset = pos_to_offset_large; | |
| levels += 4 * stride; | |
| cc += 16; | |
| row -= 4; | |
| } while (row); | |
| coeff_contexts[0] = 0; | |
| } | |
| static INLINE void get_4_nz_map_contexts_hor(const uint8_t *levels, | |
| const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = 4 + TX_PAD_HOR; | |
| const uint8x16_t pos_to_offset = | |
| vreinterpretq_u8_u32(vdupq_n_u32(SIG_COEF_CONTEXTS_2D_X4_051010)); | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(height % 4)); | |
| int row = height; | |
| do { | |
| load_levels_4x4x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset); | |
| vst1q_u8(coeff_contexts, count); | |
| levels += 4 * stride; | |
| coeff_contexts += 16; | |
| row -= 4; | |
| } while (row); | |
| } | |
| static INLINE void get_4_nz_map_contexts_ver(const uint8_t *levels, | |
| const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = 4 + TX_PAD_HOR; | |
| const uint8x16_t pos_to_offset_large = vdupq_n_u8(SIG_COEF_CONTEXTS_2D + 10); | |
| uint8x16_t pos_to_offset = vld1q_u8(c_4_po_ver); | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(height % 4)); | |
| int row = height; | |
| do { | |
| load_levels_4x4x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset); | |
| vst1q_u8(coeff_contexts, count); | |
| pos_to_offset = pos_to_offset_large; | |
| levels += 4 * stride; | |
| coeff_contexts += 16; | |
| row -= 4; | |
| } while (row); | |
| } | |
| static INLINE void get_8_coeff_contexts_2d(const uint8_t *levels, | |
| const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = 8 + TX_PAD_HOR; | |
| uint8_t *cc = coeff_contexts; | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| uint8x16_t pos_to_offset[3]; | |
| assert(!(height % 2)); | |
| if (height == 8) { | |
| pos_to_offset[0] = vld1q_u8(c_8_po_2d_8[0]); | |
| pos_to_offset[1] = vld1q_u8(c_8_po_2d_8[1]); | |
| } else if (height < 8) { | |
| pos_to_offset[0] = vld1q_u8(c_8_po_2d_l[0]); | |
| pos_to_offset[1] = vld1q_u8(c_8_po_2d_l[1]); | |
| } else { | |
| pos_to_offset[0] = vld1q_u8(c_8_po_2d_g[0]); | |
| pos_to_offset[1] = vld1q_u8(c_8_po_2d_g[1]); | |
| } | |
| pos_to_offset[2] = vdupq_n_u8(21); | |
| int row = height; | |
| do { | |
| load_levels_8x2x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset[0]); | |
| vst1q_u8(cc, count); | |
| pos_to_offset[0] = pos_to_offset[1]; | |
| pos_to_offset[1] = pos_to_offset[2]; | |
| levels += 2 * stride; | |
| cc += 16; | |
| row -= 2; | |
| } while (row); | |
| coeff_contexts[0] = 0; | |
| } | |
| static INLINE void get_8_coeff_contexts_hor(const uint8_t *levels, | |
| const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = 8 + TX_PAD_HOR; | |
| const uint8x16_t pos_to_offset = vld1q_u8(c_8_po_hor); | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(height % 2)); | |
| int row = height; | |
| do { | |
| load_levels_8x2x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset); | |
| vst1q_u8(coeff_contexts, count); | |
| levels += 2 * stride; | |
| coeff_contexts += 16; | |
| row -= 2; | |
| } while (row); | |
| } | |
| static INLINE void get_8_coeff_contexts_ver(const uint8_t *levels, | |
| const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = 8 + TX_PAD_HOR; | |
| const uint8x16_t pos_to_offset_large = vdupq_n_u8(SIG_COEF_CONTEXTS_2D + 10); | |
| uint8x16_t pos_to_offset = vcombine_u8(vdup_n_u8(SIG_COEF_CONTEXTS_2D + 0), | |
| vdup_n_u8(SIG_COEF_CONTEXTS_2D + 5)); | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(height % 2)); | |
| int row = height; | |
| do { | |
| load_levels_8x2x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset); | |
| vst1q_u8(coeff_contexts, count); | |
| pos_to_offset = pos_to_offset_large; | |
| levels += 2 * stride; | |
| coeff_contexts += 16; | |
| row -= 2; | |
| } while (row); | |
| } | |
| static INLINE void get_16n_coeff_contexts_2d(const uint8_t *levels, | |
| const int real_width, | |
| const int real_height, | |
| const int width, const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = width + TX_PAD_HOR; | |
| uint8_t *cc = coeff_contexts; | |
| int row = height; | |
| uint8x16_t pos_to_offset[5]; | |
| uint8x16_t pos_to_offset_large[3]; | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(width % 16)); | |
| pos_to_offset_large[2] = vdupq_n_u8(21); | |
| if (real_width == real_height) { | |
| pos_to_offset[0] = vld1q_u8(c_16_po_2d_e[0]); | |
| pos_to_offset[1] = vld1q_u8(c_16_po_2d_e[1]); | |
| pos_to_offset[2] = vld1q_u8(c_16_po_2d_e[2]); | |
| pos_to_offset[3] = vld1q_u8(c_16_po_2d_e[3]); | |
| pos_to_offset[4] = pos_to_offset_large[0] = pos_to_offset_large[1] = | |
| pos_to_offset_large[2]; | |
| } else if (real_width > real_height) { | |
| pos_to_offset[0] = vld1q_u8(c_16_po_2d_g[0]); | |
| pos_to_offset[1] = vld1q_u8(c_16_po_2d_g[1]); | |
| pos_to_offset[2] = pos_to_offset[3] = pos_to_offset[4] = | |
| vld1q_u8(c_16_po_2d_g[2]); | |
| pos_to_offset_large[0] = pos_to_offset_large[1] = pos_to_offset_large[2]; | |
| } else { // real_width < real_height | |
| pos_to_offset[0] = pos_to_offset[1] = vld1q_u8(c_16_po_2d_l[0]); | |
| pos_to_offset[2] = vld1q_u8(c_16_po_2d_l[1]); | |
| pos_to_offset[3] = vld1q_u8(c_16_po_2d_l[2]); | |
| pos_to_offset[4] = pos_to_offset_large[2]; | |
| pos_to_offset_large[0] = pos_to_offset_large[1] = vdupq_n_u8(11); | |
| } | |
| do { | |
| int w = width; | |
| do { | |
| load_levels_16x1x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset[0]); | |
| vst1q_u8(cc, count); | |
| levels += 16; | |
| cc += 16; | |
| w -= 16; | |
| pos_to_offset[0] = pos_to_offset_large[0]; | |
| } while (w); | |
| pos_to_offset[0] = pos_to_offset[1]; | |
| pos_to_offset[1] = pos_to_offset[2]; | |
| pos_to_offset[2] = pos_to_offset[3]; | |
| pos_to_offset[3] = pos_to_offset[4]; | |
| pos_to_offset_large[0] = pos_to_offset_large[1]; | |
| pos_to_offset_large[1] = pos_to_offset_large[2]; | |
| levels += TX_PAD_HOR; | |
| } while (--row); | |
| coeff_contexts[0] = 0; | |
| } | |
| static INLINE void get_16n_coeff_contexts_hor(const uint8_t *levels, | |
| const int width, const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = width + TX_PAD_HOR; | |
| const uint8x16_t pos_to_offset_large = vdupq_n_u8(SIG_COEF_CONTEXTS_2D + 10); | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(width % 16)); | |
| int row = height; | |
| do { | |
| uint8x16_t pos_to_offset = vld1q_u8(c_16_po_hor); | |
| int w = width; | |
| do { | |
| load_levels_16x1x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset); | |
| vst1q_u8(coeff_contexts, count); | |
| pos_to_offset = pos_to_offset_large; | |
| levels += 16; | |
| coeff_contexts += 16; | |
| w -= 16; | |
| } while (w); | |
| levels += TX_PAD_HOR; | |
| } while (--row); | |
| } | |
| static INLINE void get_16n_coeff_contexts_ver(const uint8_t *levels, | |
| const int width, const int height, | |
| const ptrdiff_t *const offsets, | |
| uint8_t *coeff_contexts) { | |
| const int stride = width + TX_PAD_HOR; | |
| uint8x16_t pos_to_offset[3]; | |
| uint8x16_t count; | |
| uint8x16_t level[5]; | |
| assert(!(width % 16)); | |
| pos_to_offset[0] = vdupq_n_u8(SIG_COEF_CONTEXTS_2D + 0); | |
| pos_to_offset[1] = vdupq_n_u8(SIG_COEF_CONTEXTS_2D + 5); | |
| pos_to_offset[2] = vdupq_n_u8(SIG_COEF_CONTEXTS_2D + 10); | |
| int row = height; | |
| do { | |
| int w = width; | |
| do { | |
| load_levels_16x1x5(levels, stride, offsets, level); | |
| count = get_coeff_contexts_kernel(level); | |
| count = vaddq_u8(count, pos_to_offset[0]); | |
| vst1q_u8(coeff_contexts, count); | |
| levels += 16; | |
| coeff_contexts += 16; | |
| w -= 16; | |
| } while (w); | |
| pos_to_offset[0] = pos_to_offset[1]; | |
| pos_to_offset[1] = pos_to_offset[2]; | |
| levels += TX_PAD_HOR; | |
| } while (--row); | |
| } | |
| // Note: levels[] must be in the range [0, 127], inclusive. | |
| void av1_get_nz_map_contexts_neon(const uint8_t *const levels, | |
| const int16_t *const scan, const uint16_t eob, | |
| const TX_SIZE tx_size, | |
| const TX_CLASS tx_class, | |
| int8_t *const coeff_contexts) { | |
| const int last_idx = eob - 1; | |
| if (!last_idx) { | |
| coeff_contexts[0] = 0; | |
| return; | |
| } | |
| uint8_t *const coefficients = (uint8_t *const)coeff_contexts; | |
| const int real_width = tx_size_wide[tx_size]; | |
| const int real_height = tx_size_high[tx_size]; | |
| const int width = get_txb_wide(tx_size); | |
| const int height = get_txb_high(tx_size); | |
| const int stride = width + TX_PAD_HOR; | |
| ptrdiff_t offsets[3]; | |
| /* coeff_contexts must be 16 byte aligned. */ | |
| assert(!((intptr_t)coeff_contexts & 0xf)); | |
| if (tx_class == TX_CLASS_2D) { | |
| offsets[0] = 0 * stride + 2; | |
| offsets[1] = 1 * stride + 1; | |
| offsets[2] = 2 * stride + 0; | |
| if (width == 4) { | |
| get_4_nz_map_contexts_2d(levels, height, offsets, coefficients); | |
| } else if (width == 8) { | |
| get_8_coeff_contexts_2d(levels, height, offsets, coefficients); | |
| } else { | |
| get_16n_coeff_contexts_2d(levels, real_width, real_height, width, height, | |
| offsets, coefficients); | |
| } | |
| } else if (tx_class == TX_CLASS_HORIZ) { | |
| offsets[0] = 2; | |
| offsets[1] = 3; | |
| offsets[2] = 4; | |
| if (width == 4) { | |
| get_4_nz_map_contexts_hor(levels, height, offsets, coefficients); | |
| } else if (width == 8) { | |
| get_8_coeff_contexts_hor(levels, height, offsets, coefficients); | |
| } else { | |
| get_16n_coeff_contexts_hor(levels, width, height, offsets, coefficients); | |
| } | |
| } else { // TX_CLASS_VERT | |
| offsets[0] = 2 * stride; | |
| offsets[1] = 3 * stride; | |
| offsets[2] = 4 * stride; | |
| if (width == 4) { | |
| get_4_nz_map_contexts_ver(levels, height, offsets, coefficients); | |
| } else if (width == 8) { | |
| get_8_coeff_contexts_ver(levels, height, offsets, coefficients); | |
| } else { | |
| get_16n_coeff_contexts_ver(levels, width, height, offsets, coefficients); | |
| } | |
| } | |
| const int bwl = get_txb_bwl(tx_size); | |
| const int pos = scan[last_idx]; | |
| if (last_idx <= (height << bwl) / 8) | |
| coeff_contexts[pos] = 1; | |
| else if (last_idx <= (height << bwl) / 4) | |
| coeff_contexts[pos] = 2; | |
| else | |
| coeff_contexts[pos] = 3; | |
| } |