av1/common/arm/wiener_convolve_neon.c

/*
 * Copyright (c) 2018, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <arm_neon.h>
#include <assert.h>

#include "config/aom_config.h"
#include "config/av1_rtcd.h"

#include "aom_dsp/txfm_common.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/transpose_neon.h"
#include "aom_ports/mem.h"
#include "av1/common/common.h"
#include "av1/common/arm/convolve_neon.h"

#define HORZ_FILTERING_CORE(t0, t1, t2, t3, t4, t5, t6, res)                 \
  res0 = vreinterpretq_s16_u16(vaddl_u8(t0, t1));                            \
  res1 = vreinterpretq_s16_u16(vaddl_u8(t2, t3));                            \
  res2 = vreinterpretq_s16_u16(vaddl_u8(t4, t5));                            \
  res3 = vreinterpretq_s16_u16(vmovl_u8(t6));                                \
  res = wiener_convolve8_horiz_8x8(res0, res1, res2, res3, filter_x_tmp, bd, \
                                   conv_params->round_0);

#define PROCESS_ROW_FOR_VERTICAL_FILTER                                      \
  __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride);                          \
                                                                             \
  do {                                                                       \
    s7 = vld1q_s16(s);                                                       \
    s += src_stride;                                                         \
                                                                             \
    t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp, \
                                   bd, conv_params->round_1);                \
    vst1_u8(d, t0);                                                          \
    d += dst_stride;                                                         \
                                                                             \
    s0 = s1;                                                                 \
    s1 = s2;                                                                 \
    s2 = s3;                                                                 \
    s3 = s4;                                                                 \
    s4 = s5;                                                                 \
    s5 = s6;                                                                 \
    s6 = s7;                                                                 \
    height--;                                                                \
  } while (height > 0);

static INLINE void process_row_for_horz_filtering(
    uint16_t *dst_ptr, int16_t *filter_x, const uint8_t *src_ptr,
    ptrdiff_t src_stride, ptrdiff_t dst_stride, int round0_bits, int w,
    int height, int bd) {
  do {
    __builtin_prefetch(src_ptr);

    uint8x8_t tt0 = vld1_u8(src_ptr);  // a0 a1 a2 a3 a4 a5 a6 a7

    __builtin_prefetch(dst_ptr);

    const uint8_t *ss = src_ptr + 8;
    uint16_t *d_tmp = dst_ptr;
    int width = w;

    do {
      uint8x8_t tt7 = vld1_u8(ss);  // a8 a9 a10 a11 a12 a13 a14 a15
      uint8x8_t ttemp_0 = tt0;
      tt0 = tt7;

      uint8x8_t tt1 = vext_u8(ttemp_0, tt7, 1);  // a1 a2 a3 a4 a5 a6 a7 a8
      uint8x8_t tt2 = vext_u8(ttemp_0, tt7, 2);  // a2 a3 a4 a5 a6 a7 a8 a9
      uint8x8_t tt3 = vext_u8(ttemp_0, tt7, 3);  // a3 a4 a5 a6 a7 a8 a9 a10
      uint8x8_t tt4 = vext_u8(ttemp_0, tt7, 4);  // a4 a5 a6 a7 a8 a9 a10 a11
      uint8x8_t tt5 = vext_u8(ttemp_0, tt7, 5);  // a5 a6 a7 a8 a9 a10 a11 a12
      uint8x8_t tt6 = vext_u8(ttemp_0, tt7, 6);  // a6 a7 a8 a9 a10 a11 a12 a13
      tt7 = vext_u8(ttemp_0, tt7, 7);            // a7 a8 a9 a10 a11 a12 a13 a14

      int16x8_t ttt0 = vreinterpretq_s16_u16(vaddl_u8(ttemp_0, tt6));
      int16x8_t ttt1 = vreinterpretq_s16_u16(vaddl_u8(tt1, tt5));
      int16x8_t ttt2 = vreinterpretq_s16_u16(vaddl_u8(tt2, tt4));
      int16x8_t ttt3 = vreinterpretq_s16_u16(vmovl_u8(tt3));
      uint16x8_t dd0 = wiener_convolve8_horiz_8x8(ttt0, ttt1, ttt2, ttt3,
                                                  filter_x, bd, round0_bits);

      vst1q_u16(d_tmp, dd0);

      ss += 8;
      d_tmp += 8;
      width -= 8;
    } while (width > 0);

    src_ptr += src_stride;
    dst_ptr += dst_stride;
    height--;
  } while (height > 0);
}

/* Wiener filter 2D
   Apply horizontal filter and store in a temporary buffer. When applying
   vertical filter, overwrite the original pixel values.
*/
void av1_wiener_convolve_add_src_neon(const uint8_t *src, ptrdiff_t src_stride,
                                      uint8_t *dst, ptrdiff_t dst_stride,
                                      const int16_t *filter_x, int x_step_q4,
                                      const int16_t *filter_y, int y_step_q4,
                                      int w, int h,
                                      const ConvolveParams *conv_params) {
  uint8_t *d;
  const uint8_t *src_ptr, *s_tmp;
  uint16_t *dst_ptr;
  (void)x_step_q4;
  (void)y_step_q4;

  int height;
  const int bd = 8;
  // Indicates the height needs to be processed during horizontal filtering.
  const int intermediate_height = h + SUBPEL_TAPS - 1;
  const int center_tap = ((SUBPEL_TAPS - 1) / 2);
  int16_t filter_x_tmp[7], filter_y_tmp[7];

  DECLARE_ALIGNED(16, uint16_t,
                  temp[(MAX_SB_SIZE + HORIZ_EXTRA_ROWS) * MAX_SB_SIZE]);

  assert(x_step_q4 == 16 && y_step_q4 == 16);
  assert(!(w % 8));

  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);

  assert(filter_x[7] == 0);
  assert(filter_y[7] == 0);

  /* assumption of horizontal filtering output will not exceed 15 bit.
     ((bd) + 1 + FILTER_BITS - conv_params->round_0) <= 15
     16 - conv_params->round_0 <= 15 -- (conv_params->round_0) >= 1
   */
  assert((conv_params->round_0) >= 1);

  memcpy(&filter_x_tmp[0], filter_x, sizeof(*filter_x) * FILTER_BITS);
  memcpy(&filter_y_tmp[0], filter_y, sizeof(*filter_y) * FILTER_BITS);

  filter_x_tmp[3] += (1 << FILTER_BITS);
  filter_y_tmp[3] += (1 << FILTER_BITS);

  s_tmp = src - center_tap * src_stride - center_tap;
  dst_ptr = temp;
  src_ptr = s_tmp;
  height = intermediate_height;

  // For aarch_64.
#if defined(__aarch64__)
  int processed_height = 0;
  uint16_t *d_tmp;
  int width, remaining_height;
  // Start of horizontal filtering.
  if (intermediate_height > 7) {
    uint16x8_t res4, res5, res6, res7, res8, res9, res10, res11;
    uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7;
    do {
      const uint8_t *s;

      __builtin_prefetch(src_ptr + 0 * src_stride);
      __builtin_prefetch(src_ptr + 1 * src_stride);
      __builtin_prefetch(src_ptr + 2 * src_stride);
      __builtin_prefetch(src_ptr + 3 * src_stride);
      __builtin_prefetch(src_ptr + 4 * src_stride);
      __builtin_prefetch(src_ptr + 5 * src_stride);
      __builtin_prefetch(src_ptr + 6 * src_stride);
      __builtin_prefetch(src_ptr + 7 * src_stride);

      load_u8_8x8(src_ptr, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
      transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);

      s = src_ptr + 7;
      d_tmp = dst_ptr;
      width = w;

      __builtin_prefetch(dst_ptr + 0 * dst_stride);
      __builtin_prefetch(dst_ptr + 1 * dst_stride);
      __builtin_prefetch(dst_ptr + 2 * dst_stride);
      __builtin_prefetch(dst_ptr + 3 * dst_stride);
      __builtin_prefetch(dst_ptr + 4 * dst_stride);
      __builtin_prefetch(dst_ptr + 5 * dst_stride);
      __builtin_prefetch(dst_ptr + 6 * dst_stride);
      __builtin_prefetch(dst_ptr + 7 * dst_stride);

      do {
        int16x8_t res0, res1, res2, res3;
        uint8x8_t t8, t9, t10, t11, t12, t13, t14;
        load_u8_8x8(s, src_stride, &t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14);
        transpose_u8_8x8(&t7, &t8, &t9, &t10, &t11, &t12, &t13, &t14);

        HORZ_FILTERING_CORE(t0, t6, t1, t5, t2, t4, t3, res4)
        HORZ_FILTERING_CORE(t1, t7, t2, t6, t3, t5, t4, res5)
        HORZ_FILTERING_CORE(t2, t8, t3, t7, t4, t6, t5, res6)
        HORZ_FILTERING_CORE(t3, t9, t4, t8, t5, t7, t6, res7)
        HORZ_FILTERING_CORE(t4, t10, t5, t9, t6, t8, t7, res8)
        HORZ_FILTERING_CORE(t5, t11, t6, t10, t7, t9, t8, res9)
        HORZ_FILTERING_CORE(t6, t12, t7, t11, t8, t10, t9, res10)
        HORZ_FILTERING_CORE(t7, t13, t8, t12, t9, t11, t10, res11)

        transpose_u16_8x8(&res4, &res5, &res6, &res7, &res8, &res9, &res10,
                          &res11);
        store_u16_8x8(d_tmp, MAX_SB_SIZE, res4, res5, res6, res7, res8, res9,
                      res10, res11);

        t0 = t8;
        t1 = t9;
        t2 = t10;
        t3 = t11;
        t4 = t12;
        t5 = t13;
        t6 = t14;
        s += 8;
        d_tmp += 8;
        width -= 8;
      } while (width > 0);
      src_ptr += 8 * src_stride;
      dst_ptr += 8 * MAX_SB_SIZE;
      height -= 8;
      processed_height += 8;
    } while (height > 7);
  }

  // Process the remaining rows for horizontal filtering.
  remaining_height = intermediate_height - processed_height;
  if (remaining_height)
    process_row_for_horz_filtering(dst_ptr, filter_x_tmp, src_ptr, src_stride,
                                   MAX_SB_SIZE, conv_params->round_0, w, height,
                                   bd);

  // Start of vertical filtering.
  {
    int16_t *src_tmp_ptr, *s;
    uint8_t *dst_tmp_ptr;
    height = h;
    width = w;
    src_tmp_ptr = (int16_t *)temp;
    dst_tmp_ptr = dst;
    src_stride = MAX_SB_SIZE;

    do {
      int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
      uint8x8_t t0;
      s = src_tmp_ptr;
      s0 = vld1q_s16(s);
      s += src_stride;
      s1 = vld1q_s16(s);
      s += src_stride;
      s2 = vld1q_s16(s);
      s += src_stride;
      s3 = vld1q_s16(s);
      s += src_stride;
      s4 = vld1q_s16(s);
      s += src_stride;
      s5 = vld1q_s16(s);
      s += src_stride;
      s6 = vld1q_s16(s);
      s += src_stride;
      d = dst_tmp_ptr;
      height = h;

      do {
        int16x8_t s8, s9, s10;
        uint8x8_t t1, t2, t3;
        __builtin_prefetch(dst_tmp_ptr + 0 * dst_stride);
        __builtin_prefetch(dst_tmp_ptr + 1 * dst_stride);
        __builtin_prefetch(dst_tmp_ptr + 2 * dst_stride);
        __builtin_prefetch(dst_tmp_ptr + 3 * dst_stride);

        s7 = vld1q_s16(s);
        s += src_stride;
        s8 = vld1q_s16(s);
        s += src_stride;
        s9 = vld1q_s16(s);
        s += src_stride;
        s10 = vld1q_s16(s);
        s += src_stride;

        t0 = wiener_convolve8_vert_4x8(s0, s1, s2, s3, s4, s5, s6, filter_y_tmp,
                                       bd, conv_params->round_1);
        t1 = wiener_convolve8_vert_4x8(s1, s2, s3, s4, s5, s6, s7, filter_y_tmp,
                                       bd, conv_params->round_1);
        t2 = wiener_convolve8_vert_4x8(s2, s3, s4, s5, s6, s7, s8, filter_y_tmp,
                                       bd, conv_params->round_1);
        t3 = wiener_convolve8_vert_4x8(s3, s4, s5, s6, s7, s8, s9, filter_y_tmp,
                                       bd, conv_params->round_1);

        vst1_u8(d, t0);
        d += dst_stride;
        vst1_u8(d, t1);
        d += dst_stride;
        vst1_u8(d, t2);
        d += dst_stride;
        vst1_u8(d, t3);
        d += dst_stride;

        s0 = s4;
        s1 = s5;
        s2 = s6;
        s3 = s7;
        s4 = s8;
        s5 = s9;
        s6 = s10;
        height -= 4;
      } while (height > 3);

      if (height) {
        PROCESS_ROW_FOR_VERTICAL_FILTER
      }
      src_tmp_ptr += 8;
      dst_tmp_ptr += 8;
      w -= 8;
    } while (w > 0);
  }
#else
  // Start of horizontal filtering.
  process_row_for_horz_filtering(dst_ptr, filter_x_tmp, src_ptr, src_stride,
                                 MAX_SB_SIZE, conv_params->round_0, w, height,
                                 bd);

  // Start of vertical filtering.
  {
    int16_t *src_tmp_ptr, *s;
    uint8_t *dst_tmp_ptr;
    src_tmp_ptr = (int16_t *)temp;
    dst_tmp_ptr = dst;
    src_stride = MAX_SB_SIZE;

    do {
      uint8x8_t t0;
      int16x8_t s0, s1, s2, s3, s4, s5, s6, s7;
      s = src_tmp_ptr;
      s0 = vld1q_s16(s);
      s += src_stride;
      s1 = vld1q_s16(s);
      s += src_stride;
      s2 = vld1q_s16(s);
      s += src_stride;
      s3 = vld1q_s16(s);
      s += src_stride;
      s4 = vld1q_s16(s);
      s += src_stride;
      s5 = vld1q_s16(s);
      s += src_stride;
      s6 = vld1q_s16(s);
      s += src_stride;
      d = dst_tmp_ptr;
      height = h;
      PROCESS_ROW_FOR_VERTICAL_FILTER

      src_tmp_ptr += 8;
      dst_tmp_ptr += 8;

      w -= 8;
    } while (w > 0);
  }
#endif
}