aom_dsp/arm/aom_convolve8_neon.h

/*
 *  Copyright (c) 2024, Alliance for Open Media. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef AOM_AOM_DSP_ARM_AOM_CONVOLVE8_NEON_H_
#define AOM_AOM_DSP_ARM_AOM_CONVOLVE8_NEON_H_

#include <arm_neon.h>

#include "config/aom_config.h"
#include "aom_dsp/arm/mem_neon.h"

static INLINE void convolve8_horiz_2tap_neon(const uint8_t *src,
                                             ptrdiff_t src_stride, uint8_t *dst,
                                             ptrdiff_t dst_stride,
                                             const int16_t *filter_x, int w,
                                             int h) {
  // Bilinear filter values are all positive.
  const uint8x8_t f0 = vdup_n_u8((uint8_t)filter_x[3]);
  const uint8x8_t f1 = vdup_n_u8((uint8_t)filter_x[4]);

  if (w == 4) {
    do {
      uint8x8_t s0 =
          load_unaligned_u8(src + 0 * src_stride + 0, (int)src_stride);
      uint8x8_t s1 =
          load_unaligned_u8(src + 0 * src_stride + 1, (int)src_stride);
      uint8x8_t s2 =
          load_unaligned_u8(src + 2 * src_stride + 0, (int)src_stride);
      uint8x8_t s3 =
          load_unaligned_u8(src + 2 * src_stride + 1, (int)src_stride);

      uint16x8_t sum0 = vmull_u8(s0, f0);
      sum0 = vmlal_u8(sum0, s1, f1);
      uint16x8_t sum1 = vmull_u8(s2, f0);
      sum1 = vmlal_u8(sum1, s3, f1);

      uint8x8_t d0 = vqrshrn_n_u16(sum0, FILTER_BITS);
      uint8x8_t d1 = vqrshrn_n_u16(sum1, FILTER_BITS);

      store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d0);
      store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d1);

      src += 4 * src_stride;
      dst += 4 * dst_stride;
      h -= 4;
    } while (h > 0);
  } else if (w == 8) {
    do {
      uint8x8_t s0 = vld1_u8(src + 0 * src_stride + 0);
      uint8x8_t s1 = vld1_u8(src + 0 * src_stride + 1);
      uint8x8_t s2 = vld1_u8(src + 1 * src_stride + 0);
      uint8x8_t s3 = vld1_u8(src + 1 * src_stride + 1);

      uint16x8_t sum0 = vmull_u8(s0, f0);
      sum0 = vmlal_u8(sum0, s1, f1);
      uint16x8_t sum1 = vmull_u8(s2, f0);
      sum1 = vmlal_u8(sum1, s3, f1);

      uint8x8_t d0 = vqrshrn_n_u16(sum0, FILTER_BITS);
      uint8x8_t d1 = vqrshrn_n_u16(sum1, FILTER_BITS);

      vst1_u8(dst + 0 * dst_stride, d0);
      vst1_u8(dst + 1 * dst_stride, d1);

      src += 2 * src_stride;
      dst += 2 * dst_stride;
      h -= 2;
    } while (h > 0);
  } else {
    do {
      int width = w;
      const uint8_t *s = src;
      uint8_t *d = dst;

      do {
        uint8x16_t s0 = vld1q_u8(s + 0);
        uint8x16_t s1 = vld1q_u8(s + 1);

        uint16x8_t sum0 = vmull_u8(vget_low_u8(s0), f0);
        sum0 = vmlal_u8(sum0, vget_low_u8(s1), f1);
        uint16x8_t sum1 = vmull_u8(vget_high_u8(s0), f0);
        sum1 = vmlal_u8(sum1, vget_high_u8(s1), f1);

        uint8x8_t d0 = vqrshrn_n_u16(sum0, FILTER_BITS);
        uint8x8_t d1 = vqrshrn_n_u16(sum1, FILTER_BITS);

        vst1q_u8(d, vcombine_u8(d0, d1));

        s += 16;
        d += 16;
        width -= 16;
      } while (width != 0);
      src += src_stride;
      dst += dst_stride;
    } while (--h > 0);
  }
}

static INLINE uint8x8_t convolve4_8(const int16x8_t s0, const int16x8_t s1,
                                    const int16x8_t s2, const int16x8_t s3,
                                    const int16x4_t filter) {
  int16x8_t sum = vmulq_lane_s16(s0, filter, 0);
  sum = vmlaq_lane_s16(sum, s1, filter, 1);
  sum = vmlaq_lane_s16(sum, s2, filter, 2);
  sum = vmlaq_lane_s16(sum, s3, filter, 3);

  // We halved the filter values so -1 from right shift.
  return vqrshrun_n_s16(sum, FILTER_BITS - 1);
}

static INLINE void convolve8_vert_4tap_neon(const uint8_t *src,
                                            ptrdiff_t src_stride, uint8_t *dst,
                                            ptrdiff_t dst_stride,
                                            const int16_t *filter_y, int w,
                                            int h) {
  // All filter values are even, halve to reduce intermediate precision
  // requirements.
  const int16x4_t filter = vshr_n_s16(vld1_s16(filter_y + 2), 1);

  if (w == 4) {
    uint8x8_t t01 = load_unaligned_u8(src + 0 * src_stride, (int)src_stride);
    uint8x8_t t12 = load_unaligned_u8(src + 1 * src_stride, (int)src_stride);

    int16x8_t s01 = vreinterpretq_s16_u16(vmovl_u8(t01));
    int16x8_t s12 = vreinterpretq_s16_u16(vmovl_u8(t12));

    src += 2 * src_stride;

    do {
      uint8x8_t t23 = load_unaligned_u8(src + 0 * src_stride, (int)src_stride);
      uint8x8_t t34 = load_unaligned_u8(src + 1 * src_stride, (int)src_stride);
      uint8x8_t t45 = load_unaligned_u8(src + 2 * src_stride, (int)src_stride);
      uint8x8_t t56 = load_unaligned_u8(src + 3 * src_stride, (int)src_stride);

      int16x8_t s23 = vreinterpretq_s16_u16(vmovl_u8(t23));
      int16x8_t s34 = vreinterpretq_s16_u16(vmovl_u8(t34));
      int16x8_t s45 = vreinterpretq_s16_u16(vmovl_u8(t45));
      int16x8_t s56 = vreinterpretq_s16_u16(vmovl_u8(t56));

      uint8x8_t d01 = convolve4_8(s01, s12, s23, s34, filter);
      uint8x8_t d23 = convolve4_8(s23, s34, s45, s56, filter);

      store_u8x4_strided_x2(dst + 0 * dst_stride, dst_stride, d01);
      store_u8x4_strided_x2(dst + 2 * dst_stride, dst_stride, d23);

      s01 = s45;
      s12 = s56;

      src += 4 * src_stride;
      dst += 4 * dst_stride;
      h -= 4;
    } while (h != 0);
  } else {
    do {
      uint8x8_t t0, t1, t2;
      load_u8_8x3(src, src_stride, &t0, &t1, &t2);

      int16x8_t s0 = vreinterpretq_s16_u16(vmovl_u8(t0));
      int16x8_t s1 = vreinterpretq_s16_u16(vmovl_u8(t1));
      int16x8_t s2 = vreinterpretq_s16_u16(vmovl_u8(t2));

      int height = h;
      const uint8_t *s = src + 3 * src_stride;
      uint8_t *d = dst;

      do {
        uint8x8_t t3;
        load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);

        int16x8_t s3 = vreinterpretq_s16_u16(vmovl_u8(t0));
        int16x8_t s4 = vreinterpretq_s16_u16(vmovl_u8(t1));
        int16x8_t s5 = vreinterpretq_s16_u16(vmovl_u8(t2));
        int16x8_t s6 = vreinterpretq_s16_u16(vmovl_u8(t3));

        uint8x8_t d0 = convolve4_8(s0, s1, s2, s3, filter);
        uint8x8_t d1 = convolve4_8(s1, s2, s3, s4, filter);
        uint8x8_t d2 = convolve4_8(s2, s3, s4, s5, filter);
        uint8x8_t d3 = convolve4_8(s3, s4, s5, s6, filter);

        store_u8_8x4(d, dst_stride, d0, d1, d2, d3);

        s0 = s4;
        s1 = s5;
        s2 = s6;

        s += 4 * src_stride;
        d += 4 * dst_stride;
        height -= 4;
      } while (height != 0);
      src += 8;
      dst += 8;
      w -= 8;
    } while (w != 0);
  }
}

#endif  // AOM_AOM_DSP_ARM_AOM_CONVOLVE8_NEON_H_