Add NEON version of av1_txb_init_levels Speedup gain size 4x4 (2.43) size 8x8 (3.09) size 16x16 (2.73) size 32x32 (2.54) size 32x32 (2.54) size 4x8 (3.43) size 8x4 (2.49) size 8x16 (3.48) size 16x8 (2.62) size 16x32 (2.55) size 32x16 (2.46) size 32x32 (2.54) size 32x32 (2.54) size 4x16 (4.51) size 16x4 (2.30) size 8x32 (3.21) size 32x8 (2.51) size 16x32 (2.55) size 32x16 (2.46) via EncodeTxbInitLevelTest*DISABLED_Speed Change-Id: If3de397647405a3e54685437dd9193ade94a9d4e
diff --git a/av1/av1.cmake b/av1/av1.cmake index d35783c..0099a21 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -377,6 +377,7 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_NEON "${AOM_ROOT}/av1/encoder/arm/neon/quantize_neon.c" "${AOM_ROOT}/av1/encoder/arm/neon/av1_error_neon.c" + "${AOM_ROOT}/av1/encoder/arm/neon/encodetxb_neon.c" "${AOM_ROOT}/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c" "${AOM_ROOT}/av1/encoder/arm/neon/highbd_fwd_txfm_neon.c")
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 18d1cd7..a9b2942 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -371,7 +371,7 @@ add_proto qw/void av1_get_nz_map_contexts/, "const uint8_t *const levels, const int16_t *const scan, const uint16_t eob, const TX_SIZE tx_size, const TX_CLASS tx_class, int8_t *const coeff_contexts"; specialize qw/av1_get_nz_map_contexts sse2/; add_proto qw/void av1_txb_init_levels/, "const tran_low_t *const coeff, const int width, const int height, uint8_t *const levels"; - specialize qw/av1_txb_init_levels sse4_1 avx2/; + specialize qw/av1_txb_init_levels sse4_1 avx2 neon/; add_proto qw/uint64_t av1_wedge_sse_from_residuals/, "const int16_t *r1, const int16_t *d, const uint8_t *m, int N"; specialize qw/av1_wedge_sse_from_residuals sse2 avx2/;
diff --git a/av1/encoder/arm/neon/encodetxb_neon.c b/av1/encoder/arm/neon/encodetxb_neon.c new file mode 100644 index 0000000..045c622 --- /dev/null +++ b/av1/encoder/arm/neon/encodetxb_neon.c
@@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> + +#include <math.h> + +#include "av1/common/txb_common.h" + +void av1_txb_init_levels_neon(const tran_low_t *const coeff, const int width, + const int height, uint8_t *const levels) { + const int stride = width + TX_PAD_HOR; + memset(levels - TX_PAD_TOP * stride, 0, + sizeof(*levels) * TX_PAD_TOP * stride); + memset(levels + stride * height, 0, + sizeof(*levels) * (TX_PAD_BOTTOM * stride + TX_PAD_END)); + + const int32x4_t zeros = vdupq_n_s32(0); + int i = 0; + uint8_t *ls = levels; + const tran_low_t *cf = coeff; + if (width == 4) { + do { + const int32x4_t coeffA = vld1q_s32(cf); + const int32x4_t coeffB = vld1q_s32(cf + width); + const int16x8_t coeffAB = + vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB)); + const int16x8_t absAB = vqabsq_s16(coeffAB); + const int8x8_t absABs = vqmovn_s16(absAB); +#if defined(__aarch64__) + const int8x16_t absAB8 = + vcombine_s8(absABs, vreinterpret_s8_s32(vget_low_s32(zeros))); + const uint8x16_t lsAB = + vreinterpretq_u8_s32(vzip1q_s32(vreinterpretq_s32_s8(absAB8), zeros)); +#else + const int32x2x2_t absAB8 = + vzip_s32(vreinterpret_s32_s8(absABs), vget_low_s32(zeros)); + const uint8x16_t lsAB = + vreinterpretq_u8_s32(vcombine_s32(absAB8.val[0], absAB8.val[1])); +#endif + vst1q_u8(ls, lsAB); + ls += (stride << 1); + cf += (width << 1); + i += 2; + } while (i < height); + } else if (width == 8) { + do { + const int32x4_t coeffA = vld1q_s32(cf); + const int32x4_t coeffB = vld1q_s32(cf + 4); + const int16x8_t coeffAB = + vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB)); + const int16x8_t absAB = vqabsq_s16(coeffAB); + const uint8x16_t absAB8 = vreinterpretq_u8_s8(vcombine_s8( + vqmovn_s16(absAB), vreinterpret_s8_s32(vget_low_s32(zeros)))); + vst1q_u8(ls, absAB8); + ls += stride; + cf += width; + i += 1; + } while (i < height); + } else { + do { + int j = 0; + do { + const int32x4_t coeffA = vld1q_s32(cf); + const int32x4_t coeffB = vld1q_s32(cf + 4); + const int32x4_t coeffC = vld1q_s32(cf + 8); + const int32x4_t coeffD = vld1q_s32(cf + 12); + const int16x8_t coeffAB = + vcombine_s16(vqmovn_s32(coeffA), vqmovn_s32(coeffB)); + const int16x8_t coeffCD = + vcombine_s16(vqmovn_s32(coeffC), vqmovn_s32(coeffD)); + const int16x8_t absAB = vqabsq_s16(coeffAB); + const int16x8_t absCD = vqabsq_s16(coeffCD); + const uint8x16_t absABCD = vreinterpretq_u8_s8( + vcombine_s8(vqmovn_s16(absAB), vqmovn_s16(absCD))); + vst1q_u8((ls + j), absABCD); + j += 16; + cf += 16; + } while (j < width); + *(int32_t *)(ls + width) = 0; + ls += stride; + i += 1; + } while (i < height); + } +}
diff --git a/test/encodetxb_test.cc b/test/encodetxb_test.cc index 385d3f1..1dc3e1f 100644 --- a/test/encodetxb_test.cc +++ b/test/encodetxb_test.cc
@@ -260,4 +260,10 @@ ::testing::Combine(::testing::Values(&av1_txb_init_levels_avx2), ::testing::Range(0, static_cast<int>(TX_SIZES_ALL), 1))); #endif +#if HAVE_NEON +INSTANTIATE_TEST_SUITE_P( + NEON, EncodeTxbInitLevelTest, + ::testing::Combine(::testing::Values(&av1_txb_init_levels_neon), + ::testing::Range(0, static_cast<int>(TX_SIZES_ALL), 1))); +#endif } // namespace