| /* |
| * Copyright (c) 2023 The WebM project authors. All Rights Reserved. |
| * Copyright (c) 2022, Alliance for Open Media. All rights reserved |
| * |
| * This source code is subject to the terms of the BSD 2 Clause License and |
| * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| * was not distributed with this source code in the LICENSE file, you can |
| * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| * Media Patent License 1.0 was not distributed with this source code in the |
| * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| */ |
| |
| #include <arm_neon.h> |
| |
| #include "config/aom_config.h" |
| #include "config/aom_dsp_rtcd.h" |
| |
| #include "aom_dsp/variance.h" |
| #include "aom_dsp/aom_filter.h" |
| #include "aom_dsp/arm/sum_neon.h" |
| |
| typedef void (*high_variance_fn_t)(const uint16_t *src, int src_stride, |
| const uint16_t *ref, int ref_stride, |
| uint32_t *sse, int *sum); |
| |
| void aom_highbd_calc16x16var_neon(const uint16_t *src, int src_stride, |
| const uint16_t *ref, int ref_stride, |
| uint32_t *sse, int *sum) { |
| int i, j; |
| int16x8_t v_sum = vdupq_n_s16(0); |
| int32x4_t v_sse_lo = vdupq_n_s32(0); |
| int32x4_t v_sse_hi = vdupq_n_s32(0); |
| |
| for (i = 0; i < 16; ++i) { |
| for (j = 0; j < 16; j += 8) { |
| const uint16x8_t v_a = vld1q_u16(&src[j]); |
| const uint16x8_t v_b = vld1q_u16(&ref[j]); |
| const int16x8_t sv_diff = vreinterpretq_s16_u16(vsubq_u16(v_a, v_b)); |
| v_sum = vaddq_s16(v_sum, sv_diff); |
| v_sse_lo = |
| vmlal_s16(v_sse_lo, vget_low_s16(sv_diff), vget_low_s16(sv_diff)); |
| v_sse_hi = |
| vmlal_s16(v_sse_hi, vget_high_s16(sv_diff), vget_high_s16(sv_diff)); |
| } |
| src += src_stride; |
| ref += ref_stride; |
| } |
| |
| *sum = horizontal_add_s16x8(v_sum); |
| *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi)); |
| } |
| |
| void aom_highbd_calc8x8var_neon(const uint16_t *src, int src_stride, |
| const uint16_t *ref, int ref_stride, |
| uint32_t *sse, int *sum) { |
| int i; |
| int16x8_t v_sum = vdupq_n_s16(0); |
| int32x4_t v_sse_lo = vdupq_n_s32(0); |
| int32x4_t v_sse_hi = vdupq_n_s32(0); |
| |
| for (i = 0; i < 8; ++i) { |
| const uint16x8_t v_a = vld1q_u16(&src[0]); |
| const uint16x8_t v_b = vld1q_u16(&ref[0]); |
| const int16x8_t sv_diff = vreinterpretq_s16_u16(vsubq_u16(v_a, v_b)); |
| v_sum = vaddq_s16(v_sum, sv_diff); |
| v_sse_lo = |
| vmlal_s16(v_sse_lo, vget_low_s16(sv_diff), vget_low_s16(sv_diff)); |
| v_sse_hi = |
| vmlal_s16(v_sse_hi, vget_high_s16(sv_diff), vget_high_s16(sv_diff)); |
| src += src_stride; |
| ref += ref_stride; |
| } |
| |
| *sum = horizontal_add_s16x8(v_sum); |
| *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi)); |
| } |
| |
| void aom_highbd_calc4x4var_neon(const uint16_t *src, int src_stride, |
| const uint16_t *ref, int ref_stride, |
| uint32_t *sse, int *sum) { |
| int i; |
| int16x8_t v_sum = vdupq_n_s16(0); |
| int32x4_t v_sse_lo = vdupq_n_s32(0); |
| int32x4_t v_sse_hi = vdupq_n_s32(0); |
| |
| for (i = 0; i < 4; i += 2) { |
| const uint16x4_t v_a_r0 = vld1_u16(&src[0]); |
| const uint16x4_t v_b_r0 = vld1_u16(&ref[0]); |
| const uint16x4_t v_a_r1 = vld1_u16(&src[src_stride]); |
| const uint16x4_t v_b_r1 = vld1_u16(&ref[ref_stride]); |
| const uint16x8_t v_a = vcombine_u16(v_a_r0, v_a_r1); |
| const uint16x8_t v_b = vcombine_u16(v_b_r0, v_b_r1); |
| const int16x8_t sv_diff = vreinterpretq_s16_u16(vsubq_u16(v_a, v_b)); |
| v_sum = vaddq_s16(v_sum, sv_diff); |
| v_sse_lo = |
| vmlal_s16(v_sse_lo, vget_low_s16(sv_diff), vget_low_s16(sv_diff)); |
| v_sse_hi = |
| vmlal_s16(v_sse_hi, vget_high_s16(sv_diff), vget_high_s16(sv_diff)); |
| src += src_stride << 1; |
| ref += ref_stride << 1; |
| } |
| |
| *sum = horizontal_add_s16x8(v_sum); |
| *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(v_sse_lo, v_sse_hi)); |
| } |
| |
| static void highbd_10_variance_neon(const uint16_t *src, int src_stride, |
| const uint16_t *ref, int ref_stride, int w, |
| int h, uint32_t *sse, int *sum, |
| high_variance_fn_t var_fn, int block_size) { |
| int i, j; |
| uint64_t sse_long = 0; |
| int32_t sum_long = 0; |
| |
| for (i = 0; i < h; i += block_size) { |
| for (j = 0; j < w; j += block_size) { |
| unsigned int sse0; |
| int sum0; |
| var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
| ref_stride, &sse0, &sum0); |
| sse_long += sse0; |
| sum_long += sum0; |
| } |
| } |
| *sum = ROUND_POWER_OF_TWO(sum_long, 2); |
| *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); |
| } |
| |
| #define VAR_FN(w, h, block_size, shift) \ |
| uint32_t aom_highbd_10_variance##w##x##h##_neon( \ |
| const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
| int ref_stride, uint32_t *sse) { \ |
| int sum; \ |
| int64_t var; \ |
| uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
| uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
| highbd_10_variance_neon( \ |
| src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
| aom_highbd_calc##block_size##x##block_size##var_neon, block_size); \ |
| var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ |
| return (var >= 0) ? (uint32_t)var : 0; \ |
| } |
| |
| VAR_FN(128, 128, 16, 14) |
| VAR_FN(128, 64, 16, 13) |
| VAR_FN(64, 128, 16, 13) |
| VAR_FN(64, 64, 16, 12) |
| VAR_FN(64, 32, 16, 11) |
| VAR_FN(32, 64, 16, 11) |
| VAR_FN(32, 32, 16, 10) |
| VAR_FN(32, 16, 16, 9) |
| VAR_FN(16, 32, 16, 9) |
| VAR_FN(16, 16, 16, 8) |
| VAR_FN(16, 8, 8, 7) |
| VAR_FN(8, 16, 8, 7) |
| VAR_FN(8, 8, 8, 6) |
| |
| VAR_FN(16, 4, 4, 6) |
| VAR_FN(4, 16, 4, 6) |
| |
| VAR_FN(8, 4, 4, 5) |
| VAR_FN(4, 8, 4, 5) |
| VAR_FN(4, 4, 4, 4) |
| |
| #if !CONFIG_REALTIME_ONLY |
| VAR_FN(64, 16, 16, 10) |
| VAR_FN(16, 64, 16, 10) |
| VAR_FN(8, 32, 8, 8) |
| VAR_FN(32, 8, 8, 8) |
| #endif // !CONFIG_REALTIME_ONLY |
| |
| #undef VAR_FN |
| |
| static INLINE uint32_t highbd_mse_wxh_neon(const uint16_t *src_ptr, |
| int src_stride, |
| const uint16_t *ref_ptr, |
| int ref_stride, int w, int h, |
| unsigned int *sse) { |
| uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; |
| |
| int i = h; |
| do { |
| int j = 0; |
| do { |
| uint16x8_t s = vld1q_u16(src_ptr + j); |
| uint16x8_t r = vld1q_u16(ref_ptr + j); |
| |
| uint16x8_t diff = vabdq_u16(s, r); |
| |
| sse_u32[0] = |
| vmlal_u16(sse_u32[0], vget_low_u16(diff), vget_low_u16(diff)); |
| sse_u32[1] = |
| vmlal_u16(sse_u32[1], vget_high_u16(diff), vget_high_u16(diff)); |
| |
| j += 8; |
| } while (j < w); |
| |
| src_ptr += src_stride; |
| ref_ptr += ref_stride; |
| } while (--i != 0); |
| |
| *sse = horizontal_add_u32x4(vaddq_u32(sse_u32[0], sse_u32[1])); |
| return *sse; |
| } |
| |
| #if defined(__ARM_FEATURE_DOTPROD) |
| |
| static INLINE uint32_t highbd_mse8_8xh_neon(const uint16_t *src_ptr, |
| int src_stride, |
| const uint16_t *ref_ptr, |
| int ref_stride, int h, |
| unsigned int *sse) { |
| uint32x4_t sse_u32 = vdupq_n_u32(0); |
| |
| int i = h / 2; |
| do { |
| uint16x8_t s0 = vld1q_u16(src_ptr); |
| src_ptr += src_stride; |
| uint16x8_t s1 = vld1q_u16(src_ptr); |
| src_ptr += src_stride; |
| uint16x8_t r0 = vld1q_u16(ref_ptr); |
| ref_ptr += ref_stride; |
| uint16x8_t r1 = vld1q_u16(ref_ptr); |
| ref_ptr += ref_stride; |
| |
| uint8x16_t s = vcombine_u8(vmovn_u16(s0), vmovn_u16(s1)); |
| uint8x16_t r = vcombine_u8(vmovn_u16(r0), vmovn_u16(r1)); |
| |
| uint8x16_t diff = vabdq_u8(s, r); |
| sse_u32 = vdotq_u32(sse_u32, diff, diff); |
| } while (--i != 0); |
| |
| *sse = horizontal_add_u32x4(sse_u32); |
| return *sse; |
| } |
| |
| static INLINE uint32_t highbd_mse8_16xh_neon(const uint16_t *src_ptr, |
| int src_stride, |
| const uint16_t *ref_ptr, |
| int ref_stride, int h, |
| unsigned int *sse) { |
| uint32x4_t sse_u32 = vdupq_n_u32(0); |
| |
| int i = h; |
| do { |
| uint16x8_t s0 = vld1q_u16(src_ptr); |
| uint16x8_t s1 = vld1q_u16(src_ptr + 8); |
| uint16x8_t r0 = vld1q_u16(ref_ptr); |
| uint16x8_t r1 = vld1q_u16(ref_ptr + 8); |
| |
| uint8x16_t s = vcombine_u8(vmovn_u16(s0), vmovn_u16(s1)); |
| uint8x16_t r = vcombine_u8(vmovn_u16(r0), vmovn_u16(r1)); |
| |
| uint8x16_t diff = vabdq_u8(s, r); |
| sse_u32 = vdotq_u32(sse_u32, diff, diff); |
| |
| src_ptr += src_stride; |
| ref_ptr += ref_stride; |
| } while (--i != 0); |
| |
| *sse = horizontal_add_u32x4(sse_u32); |
| return *sse; |
| } |
| |
| #else // !defined(__ARM_FEATURE_DOTPROD) |
| |
| static INLINE uint32_t highbd_mse8_8xh_neon(const uint16_t *src_ptr, |
| int src_stride, |
| const uint16_t *ref_ptr, |
| int ref_stride, int h, |
| unsigned int *sse) { |
| return highbd_mse_wxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 8, h, |
| sse); |
| } |
| |
| static INLINE uint32_t highbd_mse8_16xh_neon(const uint16_t *src_ptr, |
| int src_stride, |
| const uint16_t *ref_ptr, |
| int ref_stride, int h, |
| unsigned int *sse) { |
| return highbd_mse_wxh_neon(src_ptr, src_stride, ref_ptr, ref_stride, 16, h, |
| sse); |
| } |
| |
| #endif // defined(__ARM_FEATURE_DOTPROD) |
| |
| #define HIGHBD_MSE_WXH_NEON(w, h) \ |
| uint32_t aom_highbd_8_mse##w##x##h##_neon( \ |
| const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
| int ref_stride, uint32_t *sse) { \ |
| uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); \ |
| uint16_t *ref = CONVERT_TO_SHORTPTR(ref_ptr); \ |
| highbd_mse8_##w##xh_neon(src, src_stride, ref, ref_stride, h, sse); \ |
| return *sse; \ |
| } \ |
| \ |
| uint32_t aom_highbd_10_mse##w##x##h##_neon( \ |
| const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
| int ref_stride, uint32_t *sse) { \ |
| uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); \ |
| uint16_t *ref = CONVERT_TO_SHORTPTR(ref_ptr); \ |
| highbd_mse_wxh_neon(src, src_stride, ref, ref_stride, w, h, sse); \ |
| *sse = ROUND_POWER_OF_TWO(*sse, 4); \ |
| return *sse; \ |
| } \ |
| \ |
| uint32_t aom_highbd_12_mse##w##x##h##_neon( \ |
| const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
| int ref_stride, uint32_t *sse) { \ |
| uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); \ |
| uint16_t *ref = CONVERT_TO_SHORTPTR(ref_ptr); \ |
| highbd_mse_wxh_neon(src, src_stride, ref, ref_stride, w, h, sse); \ |
| *sse = ROUND_POWER_OF_TWO(*sse, 8); \ |
| return *sse; \ |
| } |
| |
| HIGHBD_MSE_WXH_NEON(16, 16) |
| HIGHBD_MSE_WXH_NEON(16, 8) |
| HIGHBD_MSE_WXH_NEON(8, 16) |
| HIGHBD_MSE_WXH_NEON(8, 8) |
| |
| #undef HIGHBD_MSE_WXH_NEON |
