Add Neon implementation for av1_highbd_calc_frame_error Add Neon implementation for av1_highbd_calc_frame_error and the corresponding tests as well. Change-Id: If452abbbede2ac1093c6b0977cc20a513f9b6d4a
diff --git a/av1/av1.cmake b/av1/av1.cmake index 4f7cdda..f5a8a9f 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -487,6 +487,7 @@ "${AOM_ROOT}/av1/encoder/x86/highbd_temporal_filter_avx2.c") list(APPEND AOM_AV1_ENCODER_INTRIN_NEON + "${AOM_ROOT}/av1/encoder/arm/neon/highbd_frame_error_neon.c" "${AOM_ROOT}/av1/encoder/arm/neon/highbd_pickrst_neon.c" "${AOM_ROOT}/av1/encoder/arm/neon/highbd_rdopt_neon.c" "${AOM_ROOT}/av1/encoder/arm/neon/highbd_temporal_filter_neon.c")
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 997777f..9941947 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -502,7 +502,7 @@ if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { add_proto qw/int64_t av1_calc_highbd_frame_error/, "const uint16_t *const ref, int ref_stride, const uint16_t *const dst, int dst_stride, int p_width, int p_height, int bd"; - specialize qw/av1_calc_highbd_frame_error sse2 avx2/; + specialize qw/av1_calc_highbd_frame_error sse2 avx2 neon/; } } }
diff --git a/av1/encoder/arm/neon/highbd_frame_error_neon.c b/av1/encoder/arm/neon/highbd_frame_error_neon.c new file mode 100644 index 0000000..58875f6 --- /dev/null +++ b/av1/encoder/arm/neon/highbd_frame_error_neon.c
@@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <arm_neon.h> +#include <assert.h> +#include <stdlib.h> + +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" + +static const uint16_t error_measure_lut_diff[257] = { + 339, 211, 181, 163, 151, 142, 136, 129, 125, 121, 117, 114, 111, 108, 107, + 104, 102, 101, 99, 97, 96, 94, 93, 92, 91, 90, 88, 88, 87, 86, + 85, 84, 84, 82, 82, 82, 80, 80, 79, 79, 78, 78, 77, 76, 76, + 76, 74, 75, 74, 73, 74, 72, 73, 71, 72, 71, 71, 70, 70, 69, + 70, 69, 68, 68, 68, 68, 67, 67, 67, 66, 67, 65, 66, 65, 65, + 65, 65, 64, 64, 64, 63, 64, 63, 63, 62, 63, 62, 62, 62, 61, + 62, 61, 61, 61, 60, 61, 60, 60, 60, 59, 60, 59, 59, 59, 59, + 58, 59, 58, 58, 58, 58, 58, 57, 58, 57, 57, 57, 56, 57, 56, + 57, 56, 56, 56, 56, 55, 56, 55, 55, 56, 55, 54, 55, 55, 54, + 55, 54, 54, 54, 54, 54, 53, 54, 53, 54, 53, 53, 53, 53, 53, + 52, 53, 52, 53, 52, 52, 52, 52, 52, 52, 52, 51, 52, 51, 51, + 52, 51, 51, 51, 50, 51, 51, 50, 51, 50, 51, 50, 50, 50, 50, + 50, 50, 49, 50, 50, 49, 50, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 48, 49, 48, 49, 48, 48, 49, 48, 48, 48, 48, 47, 48, 48, + 48, 47, 48, 47, 47, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 46, 47, 46, 47, 46, 47, 46, 46, 46, 47, 46, 46, 46, 45, 46, + 46, 46, 45, 46, 46, 45, 45, 46, 45, 45, 46, 45, 45, 45, 45, + 0 +}; + +static const int hbd_12_error_measure_lut[257] = { + 0, 5424, 8800, 11696, 14304, 16720, 18992, 21168, 23232, + 25232, 27168, 29040, 30864, 32640, 34368, 36080, 37744, 39376, + 40992, 42576, 44128, 45664, 47168, 48656, 50128, 51584, 53024, + 54432, 55840, 57232, 58608, 59968, 61312, 62656, 63968, 65280, + 66592, 67872, 69152, 70416, 71680, 72928, 74176, 75408, 76624, + 77840, 79056, 80240, 81440, 82624, 83792, 84976, 86128, 87296, + 88432, 89584, 90720, 91856, 92976, 94096, 95200, 96320, 97424, + 98512, 99600, 100688, 101776, 102848, 103920, 104992, 106048, 107120, + 108160, 109216, 110256, 111296, 112336, 113376, 114400, 115424, 116448, + 117456, 118480, 119488, 120496, 121488, 122496, 123488, 124480, 125472, + 126448, 127440, 128416, 129392, 130368, 131328, 132304, 133264, 134224, + 135184, 136128, 137088, 138032, 138976, 139920, 140864, 141792, 142736, + 143664, 144592, 145520, 146448, 147376, 148288, 149216, 150128, 151040, + 151952, 152848, 153760, 154656, 155568, 156464, 157360, 158256, 159152, + 160032, 160928, 161808, 162688, 163584, 164464, 165328, 166208, 167088, + 167952, 168832, 169696, 170560, 171424, 172288, 173152, 174000, 174864, + 175712, 176576, 177424, 178272, 179120, 179968, 180816, 181648, 182496, + 183328, 184176, 185008, 185840, 186672, 187504, 188336, 189168, 190000, + 190816, 191648, 192464, 193280, 194112, 194928, 195744, 196560, 197360, + 198176, 198992, 199792, 200608, 201408, 202224, 203024, 203824, 204624, + 205424, 206224, 207024, 207808, 208608, 209408, 210192, 210992, 211776, + 212560, 213344, 214128, 214912, 215696, 216480, 217264, 218048, 218816, + 219600, 220368, 221152, 221920, 222688, 223472, 224240, 225008, 225776, + 226544, 227296, 228064, 228832, 229600, 230352, 231120, 231872, 232624, + 233392, 234144, 234896, 235648, 236400, 237152, 237904, 238656, 239408, + 240160, 240896, 241648, 242384, 243136, 243872, 244624, 245360, 246096, + 246832, 247584, 248320, 249056, 249792, 250512, 251248, 251984, 252720, + 253440, 254176, 254912, 255632, 256352, 257088, 257808, 258528, 259264, + 259984, 260704, 261424, 262144, 262144, +}; + +static const int hbd_10_error_measure_lut[257] = { + 0, 1356, 2200, 2924, 3576, 4180, 4748, 5292, 5808, 6308, 6792, + 7260, 7716, 8160, 8592, 9020, 9436, 9844, 10248, 10644, 11032, 11416, + 11792, 12164, 12532, 12896, 13256, 13608, 13960, 14308, 14652, 14992, 15328, + 15664, 15992, 16320, 16648, 16968, 17288, 17604, 17920, 18232, 18544, 18852, + 19156, 19460, 19764, 20060, 20360, 20656, 20948, 21244, 21532, 21824, 22108, + 22396, 22680, 22964, 23244, 23524, 23800, 24080, 24356, 24628, 24900, 25172, + 25444, 25712, 25980, 26248, 26512, 26780, 27040, 27304, 27564, 27824, 28084, + 28344, 28600, 28856, 29112, 29364, 29620, 29872, 30124, 30372, 30624, 30872, + 31120, 31368, 31612, 31860, 32104, 32348, 32592, 32832, 33076, 33316, 33556, + 33796, 34032, 34272, 34508, 34744, 34980, 35216, 35448, 35684, 35916, 36148, + 36380, 36612, 36844, 37072, 37304, 37532, 37760, 37988, 38212, 38440, 38664, + 38892, 39116, 39340, 39564, 39788, 40008, 40232, 40452, 40672, 40896, 41116, + 41332, 41552, 41772, 41988, 42208, 42424, 42640, 42856, 43072, 43288, 43500, + 43716, 43928, 44144, 44356, 44568, 44780, 44992, 45204, 45412, 45624, 45832, + 46044, 46252, 46460, 46668, 46876, 47084, 47292, 47500, 47704, 47912, 48116, + 48320, 48528, 48732, 48936, 49140, 49340, 49544, 49748, 49948, 50152, 50352, + 50556, 50756, 50956, 51156, 51356, 51556, 51756, 51952, 52152, 52352, 52548, + 52748, 52944, 53140, 53336, 53532, 53728, 53924, 54120, 54316, 54512, 54704, + 54900, 55092, 55288, 55480, 55672, 55868, 56060, 56252, 56444, 56636, 56824, + 57016, 57208, 57400, 57588, 57780, 57968, 58156, 58348, 58536, 58724, 58912, + 59100, 59288, 59476, 59664, 59852, 60040, 60224, 60412, 60596, 60784, 60968, + 61156, 61340, 61524, 61708, 61896, 62080, 62264, 62448, 62628, 62812, 62996, + 63180, 63360, 63544, 63728, 63908, 64088, 64272, 64452, 64632, 64816, 64996, + 65176, 65356, 65536, 65536, +}; + +static const int hbd_8_error_measure_lut[257] = { + 0, 339, 550, 731, 894, 1045, 1187, 1323, 1452, 1577, 1698, + 1815, 1929, 2040, 2148, 2255, 2359, 2461, 2562, 2661, 2758, 2854, + 2948, 3041, 3133, 3224, 3314, 3402, 3490, 3577, 3663, 3748, 3832, + 3916, 3998, 4080, 4162, 4242, 4322, 4401, 4480, 4558, 4636, 4713, + 4789, 4865, 4941, 5015, 5090, 5164, 5237, 5311, 5383, 5456, 5527, + 5599, 5670, 5741, 5811, 5881, 5950, 6020, 6089, 6157, 6225, 6293, + 6361, 6428, 6495, 6562, 6628, 6695, 6760, 6826, 6891, 6956, 7021, + 7086, 7150, 7214, 7278, 7341, 7405, 7468, 7531, 7593, 7656, 7718, + 7780, 7842, 7903, 7965, 8026, 8087, 8148, 8208, 8269, 8329, 8389, + 8449, 8508, 8568, 8627, 8686, 8745, 8804, 8862, 8921, 8979, 9037, + 9095, 9153, 9211, 9268, 9326, 9383, 9440, 9497, 9553, 9610, 9666, + 9723, 9779, 9835, 9891, 9947, 10002, 10058, 10113, 10168, 10224, 10279, + 10333, 10388, 10443, 10497, 10552, 10606, 10660, 10714, 10768, 10822, 10875, + 10929, 10982, 11036, 11089, 11142, 11195, 11248, 11301, 11353, 11406, 11458, + 11511, 11563, 11615, 11667, 11719, 11771, 11823, 11875, 11926, 11978, 12029, + 12080, 12132, 12183, 12234, 12285, 12335, 12386, 12437, 12487, 12538, 12588, + 12639, 12689, 12739, 12789, 12839, 12889, 12939, 12988, 13038, 13088, 13137, + 13187, 13236, 13285, 13334, 13383, 13432, 13481, 13530, 13579, 13628, 13676, + 13725, 13773, 13822, 13870, 13918, 13967, 14015, 14063, 14111, 14159, 14206, + 14254, 14302, 14350, 14397, 14445, 14492, 14539, 14587, 14634, 14681, 14728, + 14775, 14822, 14869, 14916, 14963, 15010, 15056, 15103, 15149, 15196, 15242, + 15289, 15335, 15381, 15427, 15474, 15520, 15566, 15612, 15657, 15703, 15749, + 15795, 15840, 15886, 15932, 15977, 16022, 16068, 16113, 16158, 16204, 16249, + 16294, 16339, 16384, 16384, +}; + +// Split error into two parts and do an interpolated table lookup. +// To compute the table index and interpolation value, we want to calculate +// the quotient and remainder of (dst - ref) / 2^(bd - 8). +#define HBD_CALC_FRAME_ERROR(bd, offset, mask) \ + static INLINE int highbd_##bd##_error_measure(int q, int r) { \ + return (hbd_##bd##_error_measure_lut[q]) + \ + (error_measure_lut_diff[q]) * r; \ + } \ + \ + int64_t av1_calc_highbd_##bd##_frame_error_neon( \ + const uint16_t *const ref, int ref_stride, const uint16_t *const dst, \ + int dst_stride, int width, int height) { \ + int64_t sum_error[4] = { 0, 0, 0, 0 }; \ + int r = 0; \ + int d = 0; \ + \ + do { \ + int w = width; \ + int rr = r; \ + int dd = d; \ + \ + do { \ + uint16x8_t dst_v = vld1q_u16(&dst[dd]); \ + uint16x8_t ref_v = vld1q_u16(&ref[rr]); \ + \ + uint64x2_t abs_v = vreinterpretq_u64_u16(vabdq_u16(dst_v, ref_v)); \ + \ + uint64_t abs0 = vgetq_lane_u64(abs_v, 0); \ + uint64_t abs1 = vgetq_lane_u64(abs_v, 1); \ + \ + sum_error[0] += highbd_##bd##_error_measure( \ + (abs0 >> (0 + offset)) & 0xFF, (abs0 >> 0) & mask); \ + sum_error[1] += highbd_##bd##_error_measure( \ + (abs0 >> (16 + offset)) & 0xFF, (abs0 >> 16) & mask); \ + sum_error[2] += highbd_##bd##_error_measure( \ + (abs0 >> (32 + offset)) & 0xFF, (abs0 >> 32) & mask); \ + sum_error[3] += highbd_##bd##_error_measure( \ + (abs0 >> (48 + offset)) & 0xFF, (abs0 >> 48) & mask); \ + \ + sum_error[0] += highbd_##bd##_error_measure( \ + (abs1 >> (0 + offset)) & 0xFF, (abs1 >> 0) & mask); \ + sum_error[1] += highbd_##bd##_error_measure( \ + (abs1 >> (16 + offset)) & 0xFF, (abs1 >> 16) & mask); \ + sum_error[2] += highbd_##bd##_error_measure( \ + (abs1 >> (32 + offset)) & 0xFF, (abs1 >> 32) & mask); \ + sum_error[3] += highbd_##bd##_error_measure( \ + (abs1 >> (48 + offset)) & 0xFF, (abs1 >> 48) & mask); \ + \ + dd += 8; \ + rr += 8; \ + w -= 8; \ + } while (w >= 8); \ + \ + while (w-- != 0) { \ + uint16_t abs_u16 = abs(dst[dd] - ref[rr]); \ + sum_error[0] += \ + highbd_##bd##_error_measure(abs_u16 >> offset, abs_u16 & mask); \ + dd++; \ + rr++; \ + } \ + \ + r += ref_stride; \ + d += dst_stride; \ + } while (--height != 0); \ + \ + return sum_error[0] + sum_error[1] + sum_error[2] + sum_error[3]; \ + } + +// 12 bitdepth +HBD_CALC_FRAME_ERROR(12, 4, 0xF) +// 10 bitdepth +HBD_CALC_FRAME_ERROR(10, 2, 0x3) +// 8 bitdepth +HBD_CALC_FRAME_ERROR(8, 0, 0x0) + +int64_t av1_calc_highbd_frame_error_neon(const uint16_t *const ref, + int ref_stride, + const uint16_t *const dst, + int dst_stride, int width, int height, + int bd) { + switch (bd) { + case 8: + default: + return av1_calc_highbd_8_frame_error_neon(ref, ref_stride, dst, + dst_stride, width, height); + case 10: + return av1_calc_highbd_10_frame_error_neon(ref, ref_stride, dst, + dst_stride, width, height); + case 12: + return av1_calc_highbd_12_frame_error_neon(ref, ref_stride, dst, + dst_stride, width, height); + } +}
diff --git a/test/frame_error_test.cc b/test/frame_error_test.cc index 265fba6..35733d9 100644 --- a/test/frame_error_test.cc +++ b/test/frame_error_test.cc
@@ -183,7 +183,7 @@ const int kBlockHeight[] = { 480, 482, 360, 720, 1080, }; -#if HAVE_AVX2 || HAVE_SSE2 +#if HAVE_AVX2 || HAVE_SSE2 || HAVE_NEON const int kBitDepths[] = { 8, 10, 12 }; #endif typedef std::tuple<highbd_frame_error_func, int, int, int> @@ -327,6 +327,15 @@ ::testing::ValuesIn(kBitDepths))); #endif +#if HAVE_NEON +INSTANTIATE_TEST_SUITE_P( + NEON, AV1HighbdFrameErrorTest, + ::testing::Combine(::testing::Values(&av1_calc_highbd_frame_error_neon), + ::testing::ValuesIn(kBlockWidth), + ::testing::ValuesIn(kBlockHeight), + ::testing::ValuesIn(kBitDepths))); +#endif + // Check that 8-bit and 16-bit code paths give the same results for // 8-bit content typedef std::tuple<int, int> HighbdFrameErrorConsistencyParam;