aom_dsp/arm/blk_sse_sum_neon.c

/*
 * Copyright (c) 2023, Alliance for Open Media. All rights reserved.
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <arm_neon.h>
#include <assert.h>

#include "config/aom_dsp_rtcd.h"
#include "config/aom_config.h"

#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"

static inline void get_blk_sse_sum_4xh_neon(const int16_t *data, int stride,
                                            int bh, int *x_sum,
                                            int64_t *x2_sum) {
  int i = bh;
  int32x4_t sum = vdupq_n_s32(0);
  int32x4_t sse = vdupq_n_s32(0);

  do {
    int16x8_t d = vcombine_s16(vld1_s16(data), vld1_s16(data + stride));

    sum = vpadalq_s16(sum, d);

    sse = vmlal_s16(sse, vget_low_s16(d), vget_low_s16(d));
    sse = vmlal_s16(sse, vget_high_s16(d), vget_high_s16(d));

    data += 2 * stride;
    i -= 2;
  } while (i != 0);

  *x_sum = horizontal_add_s32x4(sum);
  *x2_sum = horizontal_long_add_s32x4(sse);
}

static inline void get_blk_sse_sum_8xh_neon(const int16_t *data, int stride,
                                            int bh, int *x_sum,
                                            int64_t *x2_sum) {
  int i = bh;
  int32x4_t sum = vdupq_n_s32(0);
  int32x4_t sse = vdupq_n_s32(0);

  // Input is 12-bit wide, so we can add up to 127 squared elements in a signed
  // 32-bits element. Since we're accumulating into an int32x4_t and the maximum
  // value for bh is 32, we don't have to worry about sse overflowing.

  do {
    int16x8_t d = vld1q_s16(data);

    sum = vpadalq_s16(sum, d);

    sse = vmlal_s16(sse, vget_low_s16(d), vget_low_s16(d));
    sse = vmlal_s16(sse, vget_high_s16(d), vget_high_s16(d));

    data += stride;
  } while (--i != 0);

  *x_sum = horizontal_add_s32x4(sum);
  *x2_sum = horizontal_long_add_s32x4(sse);
}

static inline void get_blk_sse_sum_large_neon(const int16_t *data, int stride,
                                              int bw, int bh, int *x_sum,
                                              int64_t *x2_sum) {
  int32x4_t sum = vdupq_n_s32(0);
  int64x2_t sse = vdupq_n_s64(0);

  // Input is 12-bit wide, so we can add up to 127 squared elements in a signed
  // 32-bits element. Since we're accumulating into an int32x4_t vector that
  // means we can process up to (127*4)/bw rows before we need to widen to
  // 64 bits.

  int i_limit = (127 * 4) / bw;
  int i_tmp = bh > i_limit ? i_limit : bh;

  int i = 0;
  do {
    int32x4_t sse_s32 = vdupq_n_s32(0);
    do {
      int j = bw;
      const int16_t *data_ptr = data;
      do {
        int16x8_t d = vld1q_s16(data_ptr);

        sum = vpadalq_s16(sum, d);

        sse_s32 = vmlal_s16(sse_s32, vget_low_s16(d), vget_low_s16(d));
        sse_s32 = vmlal_s16(sse_s32, vget_high_s16(d), vget_high_s16(d));

        data_ptr += 8;
        j -= 8;
      } while (j != 0);

      data += stride;
      i++;
    } while (i < i_tmp && i < bh);

    sse = vpadalq_s32(sse, sse_s32);
    i_tmp += i_limit;
  } while (i < bh);

  *x_sum = horizontal_add_s32x4(sum);
  *x2_sum = horizontal_add_s64x2(sse);
}

void aom_get_blk_sse_sum_neon(const int16_t *data, int stride, int bw, int bh,
                              int *x_sum, int64_t *x2_sum) {
  if (bw == 4) {
    get_blk_sse_sum_4xh_neon(data, stride, bh, x_sum, x2_sum);
  } else if (bw == 8) {
    get_blk_sse_sum_8xh_neon(data, stride, bh, x_sum, x2_sum);
  } else {
    assert(bw % 8 == 0);
    get_blk_sse_sum_large_neon(data, stride, bw, bh, x_sum, x2_sum);
  }
}