av1/encoder/clpf_rdo_simd.h

/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include "./aom_dsp_rtcd.h"
#include "aom_dsp/aom_simd.h"
#include "aom_ports/mem.h"

SIMD_INLINE void calc_diff(v128 o, v128 *a, v128 *b, v128 *c, v128 *d, v128 *e,
                           v128 *f) {
  // The difference will be 9 bit, offset by 128 so we can use saturated
  // sub to avoid going to 16 bit temporarily before "strength" clipping.
  const v128 c128 = v128_dup_8(128);
  v128 x = v128_add_8(c128, o);
  *a = v128_ssub_s8(v128_add_8(c128, *a), x);
  *b = v128_ssub_s8(v128_add_8(c128, *b), x);
  *c = v128_ssub_s8(v128_add_8(c128, *c), x);
  *d = v128_ssub_s8(v128_add_8(c128, *d), x);
  *e = v128_ssub_s8(v128_add_8(c128, *e), x);
  *f = v128_ssub_s8(v128_add_8(c128, *f), x);
}

SIMD_INLINE v128 delta_kernel(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e,
                              v128 f, v128 sp, v128 sm) {
  const v128 tmp = v128_add_8(v128_max_s8(v128_min_s8(c, sp), sm),
                              v128_max_s8(v128_min_s8(d, sp), sm));
  const v128 delta = v128_add_8(
      v128_add_8(v128_shl_8(v128_add_8(v128_max_s8(v128_min_s8(a, sp), sm),
                                       v128_max_s8(v128_min_s8(f, sp), sm)),
                            2),
                 v128_add_8(v128_max_s8(v128_min_s8(b, sp), sm),
                            v128_max_s8(v128_min_s8(e, sp), sm))),
      v128_add_8(v128_add_8(tmp, tmp), tmp));

  return v128_add_8(
      o, v128_shr_s8(
             v128_add_8(v128_dup_8(8),
                        v128_add_8(delta, v128_cmplt_s8(delta, v128_zero()))),
             4));
}

SIMD_INLINE v128 calc_delta(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e,
                            v128 f, v128 sp, v128 sm) {
  calc_diff(o, &a, &b, &c, &d, &e, &f);
  return delta_kernel(o, a, b, c, d, e, f, sp, sm);
}

SIMD_INLINE void clip_sides(v128 *b, v128 *c, v128 *d, v128 *e, int left,
                            int right) {
  DECLARE_ALIGNED(16, static const uint64_t,
                  b_shuff[]) = { 0x0504030201000000LL, 0x0d0c0b0a09080808LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  c_shuff[]) = { 0x0605040302010000LL, 0x0e0d0c0b0a090808LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  d_shuff[]) = { 0x0707060504030201LL, 0x0f0f0e0d0c0b0a09LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  e_shuff[]) = { 0x0707070605040302LL, 0x0f0f0f0e0d0c0b0aLL };

  if (!left) {  // Left clipping
    *b = v128_shuffle_8(*b, v128_load_aligned(b_shuff));
    *c = v128_shuffle_8(*c, v128_load_aligned(c_shuff));
  }
  if (!right) {  // Right clipping
    *d = v128_shuffle_8(*d, v128_load_aligned(d_shuff));
    *e = v128_shuffle_8(*e, v128_load_aligned(e_shuff));
  }
}

SIMD_INLINE void read_two_lines(const uint8_t *rec, const uint8_t *org,
                                int rstride, int ostride, int x0, int y0,
                                int bottom, int right, int y, v128 *o, v128 *r,
                                v128 *a, v128 *b, v128 *c, v128 *d, v128 *e,
                                v128 *f) {
  const v64 k1 = v64_load_aligned(org);
  const v64 k2 = v64_load_aligned(org + ostride);
  const v64 l1 = v64_load_aligned(rec);
  const v64 l2 = v64_load_aligned(rec + rstride);
  *o = v128_from_v64(k1, k2);
  *r = v128_from_v64(l1, l2);
  *a = v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1);
  *f = v128_from_v64(l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride));
  *b = v128_from_v64(v64_load_unaligned(rec - 2 * !!x0),
                     v64_load_unaligned(rec - 2 * !!x0 + rstride));
  *c = v128_from_v64(v64_load_unaligned(rec - !!x0),
                     v64_load_unaligned(rec - !!x0 + rstride));
  *d = v128_from_v64(v64_load_unaligned(rec + !!right),
                     v64_load_unaligned(rec + !!right + rstride));
  *e = v128_from_v64(v64_load_unaligned(rec + 2 * !!right),
                     v64_load_unaligned(rec + 2 * !!right + rstride));
  clip_sides(b, c, d, e, x0, right);
}

void SIMD_FUNC(aom_clpf_detect)(const uint8_t *rec, const uint8_t *org,
                                int rstride, int ostride, int x0, int y0,
                                int width, int height, int *sum0, int *sum1,
                                unsigned int strength, int size) {
  const v128 sp = v128_dup_8(strength);
  const v128 sm = v128_dup_8(-(int)strength);
  const int right = width - 8 - x0;
  const int bottom = height - 2 - y0;
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  int y;

  if (size != 8) {  // Fallback to plain C
    aom_clpf_detect_c(rec, org, rstride, ostride, x0, y0, width, height, sum0,
                      sum1, strength, size);
    return;
  }

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  for (y = 0; y < 8; y += 2) {
    v128 a, b, c, d, e, f, o, r;
    read_two_lines(rec, org, rstride, ostride, x0, y0, bottom, right, y, &o, &r,
                   &a, &b, &c, &d, &e, &f);
    ssd0 = v128_ssd_u8(ssd0, o, r);
    ssd1 = v128_ssd_u8(ssd1, o, calc_delta(r, a, b, c, d, e, f, sp, sm));
    rec += rstride * 2;
    org += ostride * 2;
  }
  *sum0 += v128_ssd_u8_sum(ssd0);
  *sum1 += v128_ssd_u8_sum(ssd1);
}

SIMD_INLINE void calc_delta_multi(v128 r, v128 o, v128 a, v128 b, v128 c,
                                  v128 d, v128 e, v128 f, ssd128_internal *ssd1,
                                  ssd128_internal *ssd2,
                                  ssd128_internal *ssd3) {
  calc_diff(r, &a, &b, &c, &d, &e, &f);
  *ssd1 = v128_ssd_u8(*ssd1, o, delta_kernel(r, a, b, c, d, e, f, v128_dup_8(1),
                                             v128_dup_8(-1)));
  *ssd2 = v128_ssd_u8(*ssd2, o, delta_kernel(r, a, b, c, d, e, f, v128_dup_8(2),
                                             v128_dup_8(-2)));
  *ssd3 = v128_ssd_u8(*ssd3, o, delta_kernel(r, a, b, c, d, e, f, v128_dup_8(4),
                                             v128_dup_8(-4)));
}

// Test multiple filter strengths at once.
void SIMD_FUNC(aom_clpf_detect_multi)(const uint8_t *rec, const uint8_t *org,
                                      int rstride, int ostride, int x0, int y0,
                                      int width, int height, int *sum,
                                      int size) {
  const int bottom = height - 2 - y0;
  const int right = width - 8 - x0;
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  ssd128_internal ssd2 = v128_ssd_u8_init();
  ssd128_internal ssd3 = v128_ssd_u8_init();
  int y;

  if (size != 8) {  // Fallback to plain C
    aom_clpf_detect_multi_c(rec, org, rstride, ostride, x0, y0, width, height,
                            sum, size);
    return;
  }

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  for (y = 0; y < 8; y += 2) {
    v128 a, b, c, d, e, f, o, r;
    read_two_lines(rec, org, rstride, ostride, x0, y0, bottom, right, y, &o, &r,
                   &a, &b, &c, &d, &e, &f);
    ssd0 = v128_ssd_u8(ssd0, o, r);
    calc_delta_multi(r, o, a, b, c, d, e, f, &ssd1, &ssd2, &ssd3);
    rec += 2 * rstride;
    org += 2 * ostride;
  }
  sum[0] += v128_ssd_u8_sum(ssd0);
  sum[1] += v128_ssd_u8_sum(ssd1);
  sum[2] += v128_ssd_u8_sum(ssd2);
  sum[3] += v128_ssd_u8_sum(ssd3);
}

#if CONFIG_AOM_HIGHBITDEPTH
SIMD_INLINE void read_two_lines_hbd(const uint16_t *rec, const uint16_t *org,
                                    int rstride, int ostride, int x0, int y0,
                                    int bottom, int right, int y, v128 *o,
                                    v128 *r, v128 *a, v128 *b, v128 *c, v128 *d,
                                    v128 *e, v128 *f, int shift) {
  const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
  const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
  *o = v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                      v128_shr_u16(v128_load_aligned(org + ostride), shift));
  *r = v128_unziplo_8(n1, n2);
  *a = v128_unziplo_8(
      v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride), shift), n1);
  *f = v128_unziplo_8(
      n2, v128_shr_u16(v128_load_unaligned(rec + ((y != bottom) + 1) * rstride),
                       shift));
  *b = v128_unziplo_8(
      v128_shr_u16(v128_load_unaligned(rec - 2 * !!x0), shift),
      v128_shr_u16(v128_load_unaligned(rec - 2 * !!x0 + rstride), shift));
  *c = v128_unziplo_8(
      v128_shr_u16(v128_load_unaligned(rec - !!x0), shift),
      v128_shr_u16(v128_load_unaligned(rec - !!x0 + rstride), shift));
  *d = v128_unziplo_8(
      v128_shr_u16(v128_load_unaligned(rec + !!right), shift),
      v128_shr_u16(v128_load_unaligned(rec + !!right + rstride), shift));
  *e = v128_unziplo_8(
      v128_shr_u16(v128_load_unaligned(rec + 2 * !!right), shift),
      v128_shr_u16(v128_load_unaligned(rec + 2 * !!right + rstride), shift));
  clip_sides(b, c, d, e, x0, right);
}

void SIMD_FUNC(aom_clpf_detect_hbd)(const uint16_t *rec, const uint16_t *org,
                                    int rstride, int ostride, int x0, int y0,
                                    int width, int height, int *sum0, int *sum1,
                                    unsigned int strength, int shift,
                                    int size) {
  const v128 sp = v128_dup_8(strength >> shift);
  const v128 sm = v128_dup_8(-(int)(strength >> shift));
  const int bottom = height - 2 - y0;
  const int right = width - 8 - x0;
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  int y;

  if (size != 8) {  // Fallback to plain C
    aom_clpf_detect_hbd_c(rec, org, rstride, ostride, x0, y0, width, height,
                          sum0, sum1, strength, shift, size);
    return;
  }

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  for (y = 0; y < 8; y += 2) {
    v128 a, b, c, d, e, f, o, r;
    read_two_lines_hbd(rec, org, rstride, ostride, x0, y0, bottom, right, y, &o,
                       &r, &a, &b, &c, &d, &e, &f, shift);
    ssd0 = v128_ssd_u8(ssd0, o, r);
    ssd1 = v128_ssd_u8(ssd1, o, calc_delta(r, a, b, c, d, e, f, sp, sm));
    rec += rstride * 2;
    org += ostride * 2;
  }
  *sum0 += v128_ssd_u8_sum(ssd0);
  *sum1 += v128_ssd_u8_sum(ssd1);
}

void SIMD_FUNC(aom_clpf_detect_multi_hbd)(const uint16_t *rec,
                                          const uint16_t *org, int rstride,
                                          int ostride, int x0, int y0,
                                          int width, int height, int *sum,
                                          int shift, int size) {
  const int bottom = height - 2 - y0;
  const int right = width - 8 - x0;
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  ssd128_internal ssd2 = v128_ssd_u8_init();
  ssd128_internal ssd3 = v128_ssd_u8_init();
  int y;

  if (size != 8) {  // Fallback to plain C
    aom_clpf_detect_multi_hbd_c(rec, org, rstride, ostride, x0, y0, width,
                                height, sum, shift, size);
    return;
  }

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  for (y = 0; y < 8; y += 2) {
    v128 a, b, c, d, e, f, o, r;
    read_two_lines_hbd(rec, org, rstride, ostride, x0, y0, bottom, right, y, &o,
                       &r, &a, &b, &c, &d, &e, &f, shift);
    ssd0 = v128_ssd_u8(ssd0, o, r);
    calc_delta_multi(r, o, a, b, c, d, e, f, &ssd1, &ssd2, &ssd3);
    rec += 2 * rstride;
    org += 2 * ostride;
  }
  sum[0] += v128_ssd_u8_sum(ssd0);
  sum[1] += v128_ssd_u8_sum(ssd1);
  sum[2] += v128_ssd_u8_sum(ssd2);
  sum[3] += v128_ssd_u8_sum(ssd3);
}
#endif