AArch64: Clean up duplicate constant tables for HBD convolutions Refactor duplicate constant tables used by various high bit-depth convolution functions so they are defined just once in a single .c file, and referenced by other files. Rename the tables which conflict with the standard bit-depth ones. Change-Id: I9d93297527913c3a48123b53a659d13622080734
diff --git a/aom_dsp/arm/highbd_convolve8_sve.c b/aom_dsp/arm/highbd_convolve8_sve.c index 882e360..1863aef 100644 --- a/aom_dsp/arm/highbd_convolve8_sve.c +++ b/aom_dsp/arm/highbd_convolve8_sve.c
@@ -22,6 +22,26 @@ #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/transpose_neon.h" +DECLARE_ALIGNED(16, const uint16_t, kHbdDotProdTbl[32]) = { + 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, + 4, 5, 6, 7, 5, 6, 7, 0, 6, 7, 0, 1, 7, 0, 1, 2, +}; + +// clang-format off +DECLARE_ALIGNED(16, const uint16_t, kDeinterleaveTbl[8]) = { + 0, 2, 4, 6, 1, 3, 5, 7, +}; +// clang-format on + +DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { + // Shift left and insert new last column in transposed 4x4 block. + 2, 3, 4, 5, 6, 7, 16, 17, 10, 11, 12, 13, 14, 15, 24, 25, + // Shift left and insert two new columns in transposed 4x4 block. + 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15, 24, 25, 26, 27, + // Shift left and insert three new columns in transposed 4x4 block. + 6, 7, 16, 17, 18, 19, 20, 21, 14, 15, 24, 25, 26, 27, 28, 29 +}; + static inline uint16x4_t highbd_convolve8_4_h(int16x8_t s[4], int16x8_t filter, uint16x4_t max) { int64x2_t sum[4]; @@ -131,16 +151,6 @@ } } -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDotProdTbl[16]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, -}; - -DECLARE_ALIGNED(16, static const uint16_t, kDeinterleaveTbl[8]) = { - 0, 2, 4, 6, 1, 3, 5, 7, -}; -// clang-format on - static inline uint16x4_t highbd_convolve4_4_h(int16x8_t s, int16x8_t filter, uint16x8x2_t permute_tbl, uint16x4_t max) { @@ -184,7 +194,7 @@ if (width == 4) { const uint16x4_t max = vdup_n_u16((1 << bd) - 1); - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)src; uint16_t *d = dst; @@ -268,15 +278,6 @@ } } -DECLARE_ALIGNED(16, static const uint8_t, kDotProdMergeBlockTbl[48]) = { - // Shift left and insert new last column in transposed 4x4 block. - 2, 3, 4, 5, 6, 7, 16, 17, 10, 11, 12, 13, 14, 15, 24, 25, - // Shift left and insert two new columns in transposed 4x4 block. - 4, 5, 6, 7, 16, 17, 18, 19, 12, 13, 14, 15, 24, 25, 26, 27, - // Shift left and insert three new columns in transposed 4x4 block. - 6, 7, 16, 17, 18, 19, 20, 21, 14, 15, 24, 25, 26, 27, 28, 29 -}; - static inline void aom_tbl2x4_s16(int16x8_t t0[4], int16x8_t t1[4], uint8x16_t tbl, int16x8_t res[4]) { int8x16x2_t samples0 = { vreinterpretq_s8_s16(t0[0]),
diff --git a/aom_dsp/arm/highbd_convolve8_sve.h b/aom_dsp/arm/highbd_convolve8_sve.h new file mode 100644 index 0000000..a28b1b1 --- /dev/null +++ b/aom_dsp/arm/highbd_convolve8_sve.h
@@ -0,0 +1,20 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifndef AOM_AOM_DSP_ARM_HIGHBD_CONVOLVE_SVE2_H_ +#define AOM_AOM_DSP_ARM_HIGHBD_CONVOLVE_SVE2_H_ + +#include "aom_ports/mem.h" + +DECLARE_ALIGNED(16, extern const uint16_t, kHbdDotProdTbl[32]); +DECLARE_ALIGNED(16, extern const uint16_t, kDeinterleaveTbl[8]); + +#endif // AOM_AOM_DSP_ARM_HIGHBD_CONVOLVE_SVE2_H_
diff --git a/av1/common/arm/convolve_sve2.c b/av1/common/arm/convolve_sve2.c index 3cda7d7..8090b24 100644 --- a/av1/common/arm/convolve_sve2.c +++ b/av1/common/arm/convolve_sve2.c
@@ -52,7 +52,7 @@ const int bd = 8; const int16x8_t sub_const = vdupq_n_s16(1 << (bd - 1)); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 =
diff --git a/av1/common/arm/highbd_compound_convolve_sve2.c b/av1/common/arm/highbd_compound_convolve_sve2.c index 493f218..36c0ae7 100644 --- a/av1/common/arm/highbd_compound_convolve_sve2.c +++ b/av1/common/arm/highbd_compound_convolve_sve2.c
@@ -18,6 +18,7 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/aom_neon_sve_bridge.h" #include "aom_dsp/arm/aom_neon_sve2_bridge.h" +#include "aom_dsp/arm/highbd_convolve8_sve.h" #include "aom_dsp/arm/mem_neon.h" #include "aom_ports/mem.h" #include "av1/common/convolve.h" @@ -27,11 +28,6 @@ #include "av1/common/arm/highbd_convolve_neon.h" #include "av1/common/arm/highbd_convolve_sve2.h" -DECLARE_ALIGNED(16, static const uint16_t, kDotProdTbl[32]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 0, 6, 7, 0, 1, 7, 0, 1, 2, -}; - static inline uint16x8_t highbd_12_convolve8_8_x(int16x8_t s0[8], int16x8_t filter, int64x2_t offset) { @@ -165,12 +161,6 @@ } while (height != 0); } -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDeinterleaveTbl[8]) = { - 0, 2, 4, 6, 1, 3, 5, 7, -}; -// clang-format on - static inline uint16x4_t highbd_12_convolve4_4_x(int16x8_t s0, int16x8_t filter, int64x2_t offset, uint16x8x2_t permute_tbl) { @@ -212,7 +202,7 @@ const int16x8_t filter = vcombine_s16(x_filter, vdup_n_s16(0)); if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -304,7 +294,7 @@ const int16x8_t filter = vcombine_s16(x_filter, vdup_n_s16(0)); if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -483,7 +473,7 @@ vdupq_n_s64((1 << (12 + FILTER_BITS)) + (1 << (12 + FILTER_BITS - 1))); const int16x8_t y_filter = vld1q_s16(y_filter_ptr); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -659,7 +649,7 @@ vdupq_n_s64((1 << (bd + FILTER_BITS)) + (1 << (bd + FILTER_BITS - 1))); const int16x8_t y_filter = vld1q_s16(y_filter_ptr); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -989,7 +979,7 @@ // a time and then process the last 3 rows separately. if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -1084,7 +1074,7 @@ // a time and then process the last 3 rows separately. if (width == 4) { - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -1210,7 +1200,7 @@ const int16x8_t y_filter = vld1q_s16(y_filter_ptr); const int64x2_t offset_s64 = vdupq_n_s64(offset); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 =
diff --git a/av1/common/arm/highbd_convolve_sve2.c b/av1/common/arm/highbd_convolve_sve2.c index 8ce6021..d50e2d0 100644 --- a/av1/common/arm/highbd_convolve_sve2.c +++ b/av1/common/arm/highbd_convolve_sve2.c
@@ -18,6 +18,7 @@ #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/aom_neon_sve_bridge.h" #include "aom_dsp/arm/aom_neon_sve2_bridge.h" +#include "aom_dsp/arm/highbd_convolve8_sve.h" #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/transpose_neon.h" #include "aom_ports/mem.h" @@ -25,10 +26,16 @@ #include "av1/common/filter.h" #include "av1/common/arm/highbd_convolve_sve2.h" -DECLARE_ALIGNED(16, static const uint16_t, kDotProdTbl[32]) = { - 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, - 4, 5, 6, 7, 5, 6, 7, 0, 6, 7, 0, 1, 7, 0, 1, 2, +// clang-format off +DECLARE_ALIGNED(16, const uint16_t, kHbdDotProdMergeBlockTbl[24]) = { + // Shift left and insert new last column in transposed 4x4 block. + 1, 2, 3, 0, 5, 6, 7, 4, + // Shift left and insert two new columns in transposed 4x4 block. + 2, 3, 0, 1, 6, 7, 4, 5, + // Shift left and insert three new columns in transposed 4x4 block. + 3, 0, 1, 2, 7, 4, 5, 6, }; +// clang-format on static inline uint16x4_t convolve12_4_x( int16x8_t s0, int16x8_t s1, int16x8_t filter_0_7, int16x8_t filter_4_11, @@ -111,7 +118,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x4_t permute_tbl = vld1q_u16_x4(kDotProdTbl); + uint16x8x4_t permute_tbl = vld1q_u16_x4(kHbdDotProdTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = vreinterpretq_u16_u64(vcombine_u64( @@ -253,12 +260,6 @@ } while (height != 0); } -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDeinterleaveTbl[8]) = { - 0, 2, 4, 6, 1, 3, 5, 7, -}; -// clang-format on - static inline uint16x4_t convolve4_4_x(int16x8_t s0, int16x8_t filter, int64x2_t offset, uint16x8x2_t permute_tbl, @@ -305,7 +306,7 @@ if (width == 4) { const uint16x4_t max = vdup_n_u16((1 << bd) - 1); - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); const int16_t *s = (const int16_t *)(src); @@ -428,7 +429,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -572,7 +573,7 @@ const int16x8_t y_filter = vld1q_s16(filter_y); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -945,7 +946,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x4_t permute_tbl = vld1q_u16_x4(kDotProdTbl); + uint16x8x4_t permute_tbl = vld1q_u16_x4(kHbdDotProdTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = vreinterpretq_u16_u64(vcombine_u64( @@ -1132,7 +1133,7 @@ if (width == 4) { const int16_t *s = (const int16_t *)(src); - uint16x8x2_t permute_tbl = vld1q_u16_x2(kDotProdTbl); + uint16x8x2_t permute_tbl = vld1q_u16_x2(kHbdDotProdTbl); do { int16x8_t s0, s1, s2, s3; @@ -1215,7 +1216,7 @@ const int16x8_t y_filter_0_7 = vld1q_s16(y_filter_ptr); const int16x8_t y_filter_4_11 = vld1q_s16(y_filter_ptr + 4); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 = @@ -1358,7 +1359,7 @@ const int32x4_t shift = vdupq_n_s32(-conv_params->round_1); const int16x8_t y_filter = vld1q_s16(filter_y); - uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kDotProdMergeBlockTbl); + uint16x8x3_t merge_block_tbl = vld1q_u16_x3(kHbdDotProdMergeBlockTbl); // Scale indices by size of the true vector length to avoid reading from an // 'undefined' portion of a vector on a system with SVE vectors > 128-bit. uint16x8_t correction0 =
diff --git a/av1/common/arm/highbd_convolve_sve2.h b/av1/common/arm/highbd_convolve_sve2.h index 40ba2cd..5862e68 100644 --- a/av1/common/arm/highbd_convolve_sve2.h +++ b/av1/common/arm/highbd_convolve_sve2.h
@@ -16,16 +16,7 @@ #include "aom_dsp/arm/aom_neon_sve2_bridge.h" -// clang-format off -DECLARE_ALIGNED(16, static const uint16_t, kDotProdMergeBlockTbl[24]) = { - // Shift left and insert new last column in transposed 4x4 block. - 1, 2, 3, 0, 5, 6, 7, 4, - // Shift left and insert two new columns in transposed 4x4 block. - 2, 3, 0, 1, 6, 7, 4, 5, - // Shift left and insert three new columns in transposed 4x4 block. - 3, 0, 1, 2, 7, 4, 5, 6, -}; -// clang-format on +DECLARE_ALIGNED(16, extern const uint16_t, kHbdDotProdMergeBlockTbl[24]); static inline void aom_tbl2x4_s16(int16x8_t t0[4], int16x8_t t1[4], uint16x8_t tbl, int16x8_t res[4]) {