/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include "aom_dsp/aom_simd.h"

SIMD_INLINE v128 calc_delta(v128 x, v128 a, v128 b, v128 c, v128 d, v128 e,
                            v128 f, v128 sp, v128 sm) {
  const v128 tmp =
      v128_add_8(v128_max_s8(v128_min_s8(v128_ssub_s8(c, x), sp), sm),
                 v128_max_s8(v128_min_s8(v128_ssub_s8(d, x), sp), sm));
  v128 delta = v128_add_8(
      v128_add_8(
          v128_shl_8(
              v128_add_8(v128_max_s8(v128_min_s8(v128_ssub_s8(a, x), sp), sm),
                         v128_max_s8(v128_min_s8(v128_ssub_s8(f, x), sp), sm)),
              2),
          v128_add_8(v128_max_s8(v128_min_s8(v128_ssub_s8(b, x), sp), sm),
                     v128_max_s8(v128_min_s8(v128_ssub_s8(e, x), sp), sm))),
      v128_add_8(v128_add_8(tmp, tmp), tmp));

  return v128_shr_s8(
      v128_add_8(v128_dup_8(8),
                 v128_add_8(delta, v128_cmplt_s8(delta, v128_zero()))),
      4);
}

void SIMD_FUNC(aom_clpf_detect)(const uint8_t *rec, const uint8_t *org,
                                int rstride, int ostride, int x0, int y0,
                                int width, int height, int *sum0, int *sum1,
                                unsigned int strength) {
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  const v128 c128 = v128_dup_8(128);
  const v128 sp = v128_dup_8(strength);
  const v128 sm = v128_dup_8(-(int)strength);
  const int bottom = height - 2 - y0;

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  if (!x0) {  // Clip left
    const v128 b_shuff = v128_from_v64(v64_from_64(0x0d0c0b0a09080808LL),
                                       v64_from_64(0x0504030201000000LL));
    const v128 c_shuff = v128_from_v64(v64_from_64(0x0e0d0c0b0a090808LL),
                                       v64_from_64(0x0605040302010000LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v64 k1 = v64_load_aligned(org);
      const v64 k2 = v64_load_aligned(org + ostride);
      const v64 l1 = v64_load_aligned(rec);
      const v64 l2 = v64_load_aligned(rec + rstride);
      v128 o = v128_from_v64(k1, k2);
      const v128 q = v128_from_v64(l1, l2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128,
          v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1));
      const v128 b = v128_shuffle_8(x, b_shuff);
      const v128 c = v128_shuffle_8(x, c_shuff);
      const v128 d = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec + 1),
                              v64_load_unaligned(rec + 1 + rstride)));
      const v128 e = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec + 2),
                              v64_load_unaligned(rec + 2 + rstride)));
      const v128 f = v128_add_8(
          c128, v128_from_v64(
                    l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      ssd1 = v128_ssd_u8(
          ssd1, o, v128_add_8(q, calc_delta(x, a, b, c, d, e, f, sp, sm)));
      rec += rstride * 2;
      org += ostride * 2;
    }
  } else if (!(width - x0 - 8)) {  // Clip right
    const v128 d_shuff = v128_from_v64(v64_from_64(0x0f0f0e0d0c0b0a09LL),
                                       v64_from_64(0x0707060504030201LL));
    const v128 e_shuff = v128_from_v64(v64_from_64(0x0f0f0f0e0d0c0b0aLL),
                                       v64_from_64(0x0707070605040302LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v64 k1 = v64_load_aligned(org);
      const v64 k2 = v64_load_aligned(org + ostride);
      const v64 l1 = v64_load_aligned(rec);
      const v64 l2 = v64_load_aligned(rec + rstride);
      v128 o = v128_from_v64(k1, k2);
      const v128 q = v128_from_v64(l1, l2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128,
          v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1));
      const v128 b = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec - 2),
                              v64_load_unaligned(rec - 2 + rstride)));
      const v128 c = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec - 1),
                              v64_load_unaligned(rec - 1 + rstride)));
      const v128 d = v128_shuffle_8(x, d_shuff);
      const v128 e = v128_shuffle_8(x, e_shuff);
      const v128 f = v128_add_8(
          c128, v128_from_v64(
                    l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      ssd1 = v128_ssd_u8(
          ssd1, o, v128_add_8(q, calc_delta(x, a, b, c, d, e, f, sp, sm)));
      rec += rstride * 2;
      org += ostride * 2;
    }
  } else {  // No left/right clipping
    int y;
    for (y = 0; y < 8; y += 2) {
      const v64 k1 = v64_load_aligned(org);
      const v64 k2 = v64_load_aligned(org + ostride);
      const v64 l1 = v64_load_aligned(rec);
      const v64 l2 = v64_load_aligned(rec + rstride);
      v128 o = v128_from_v64(k1, k2);
      const v128 q = v128_from_v64(l1, l2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128,
          v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1));
      const v128 b = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec - 2),
                              v64_load_unaligned(rec - 2 + rstride)));
      const v128 c = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec - 1),
                              v64_load_unaligned(rec - 1 + rstride)));
      const v128 d = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec + 1),
                              v64_load_unaligned(rec + 1 + rstride)));
      const v128 e = v128_add_8(
          c128, v128_from_v64(v64_load_unaligned(rec + 2),
                              v64_load_unaligned(rec + 2 + rstride)));
      const v128 f = v128_add_8(
          c128, v128_from_v64(
                    l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      ssd1 = v128_ssd_u8(
          ssd1, o, v128_add_8(q, calc_delta(x, a, b, c, d, e, f, sp, sm)));
      rec += rstride * 2;
      org += ostride * 2;
    }
  }
  *sum0 += v128_ssd_u8_sum(ssd0);
  *sum1 += v128_ssd_u8_sum(ssd1);
}

SIMD_INLINE void calc_delta_multi(v128 x, v128 q, v128 o, v128 a, v128 b,
                                  v128 c, v128 d, v128 e, v128 f, v128 cp1,
                                  v128 cm1, v128 cp2, v128 cm2, v128 cp4,
                                  v128 cm4, ssd128_internal *ssd1,
                                  ssd128_internal *ssd2,
                                  ssd128_internal *ssd3) {
  v128 tmp, delta1, delta2, delta3;
  const v128 c8 = v128_dup_8(8);

  a = v128_ssub_s8(a, x);
  b = v128_ssub_s8(b, x);
  c = v128_ssub_s8(c, x);
  d = v128_ssub_s8(d, x);
  e = v128_ssub_s8(e, x);
  f = v128_ssub_s8(f, x);
  tmp = v128_add_8(v128_max_s8(v128_min_s8(c, cp1), cm1),
                   v128_max_s8(v128_min_s8(d, cp1), cm1));
  delta1 = v128_add_8(
      v128_add_8(v128_shl_8(v128_add_8(v128_max_s8(v128_min_s8(a, cp1), cm1),
                                       v128_max_s8(v128_min_s8(f, cp1), cm1)),
                            2),
                 v128_add_8(v128_max_s8(v128_min_s8(b, cp1), cm1),
                            v128_max_s8(v128_min_s8(e, cp1), cm1))),
      v128_add_8(v128_add_8(tmp, tmp), tmp));
  tmp = v128_add_8(v128_max_s8(v128_min_s8(c, cp2), cm2),
                   v128_max_s8(v128_min_s8(d, cp2), cm2));
  delta2 = v128_add_8(
      v128_add_8(v128_shl_8(v128_add_8(v128_max_s8(v128_min_s8(a, cp2), cm2),
                                       v128_max_s8(v128_min_s8(f, cp2), cm2)),
                            2),
                 v128_add_8(v128_max_s8(v128_min_s8(b, cp2), cm2),
                            v128_max_s8(v128_min_s8(e, cp2), cm2))),
      v128_add_8(v128_add_8(tmp, tmp), tmp));
  tmp = v128_add_8(v128_max_s8(v128_min_s8(c, cp4), cm4),
                   v128_max_s8(v128_min_s8(d, cp4), cm4));
  delta3 = v128_add_8(
      v128_add_8(v128_shl_8(v128_add_8(v128_max_s8(v128_min_s8(a, cp4), cm4),
                                       v128_max_s8(v128_min_s8(f, cp4), cm4)),
                            2),
                 v128_add_8(v128_max_s8(v128_min_s8(b, cp4), cm4),
                            v128_max_s8(v128_min_s8(e, cp4), cm4))),
      v128_add_8(v128_add_8(tmp, tmp), tmp));

  *ssd1 = v128_ssd_u8(
      *ssd1, o,
      v128_add_8(
          q, v128_shr_s8(
                 v128_add_8(c8, v128_add_8(delta1,
                                           v128_cmplt_s8(delta1, v128_zero()))),
                 4)));
  *ssd2 = v128_ssd_u8(
      *ssd2, o,
      v128_add_8(
          q, v128_shr_s8(
                 v128_add_8(c8, v128_add_8(delta2,
                                           v128_cmplt_s8(delta2, v128_zero()))),
                 4)));
  *ssd3 = v128_ssd_u8(
      *ssd3, o,
      v128_add_8(
          q, v128_shr_s8(
                 v128_add_8(c8, v128_add_8(delta3,
                                           v128_cmplt_s8(delta3, v128_zero()))),
                 4)));
}

// Test multiple filter strengths at once.
void SIMD_FUNC(aom_clpf_detect_multi)(const uint8_t *rec, const uint8_t *org,
                                      int rstride, int ostride, int x0, int y0,
                                      int width, int height, int *sum) {
  const v128 c128 = v128_dup_8(128);
  const v128 cp1 = v128_dup_8(1);
  const v128 cm1 = v128_dup_8(-1);
  const v128 cp2 = v128_dup_8(2);
  const v128 cm2 = v128_dup_8(-2);
  const v128 cp4 = v128_dup_8(4);
  const v128 cm4 = v128_dup_8(-4);
  const int bottom = height - 2 - y0;
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  ssd128_internal ssd2 = v128_ssd_u8_init();
  ssd128_internal ssd3 = v128_ssd_u8_init();

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  if (!x0) {  // Clip left
    const v128 b_shuff = v128_from_v64(v64_from_64(0x0d0c0b0a09080808LL),
                                       v64_from_64(0x0504030201000000LL));
    const v128 c_shuff = v128_from_v64(v64_from_64(0x0e0d0c0b0a090808LL),
                                       v64_from_64(0x0605040302010000LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v64 k1 = v64_load_aligned(org);
      const v64 k2 = v64_load_aligned(org + ostride);
      const v64 l1 = v64_load_aligned(rec);
      const v64 l2 = v64_load_aligned(rec + rstride);
      v128 o = v128_from_v64(k1, k2);
      const v128 q = v128_from_v64(l1, l2);
      const v128 x = v128_add_8(c128, q);
      v128 a = v128_add_8(
          c128,
          v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1));
      v128 b = v128_shuffle_8(x, b_shuff);
      v128 c = v128_shuffle_8(x, c_shuff);
      v128 d = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec + 1),
                                        v64_load_unaligned(rec + 1 + rstride)));
      v128 e = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec + 2),
                                        v64_load_unaligned(rec + 2 + rstride)));
      v128 f = v128_add_8(
          c128, v128_from_v64(
                    l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      calc_delta_multi(x, q, o, a, b, c, d, e, f, cp1, cm1, cp2, cm2, cp4, cm4,
                       &ssd1, &ssd2, &ssd3);
      rec += 2 * rstride;
      org += 2 * ostride;
    }
  } else if (!(width - x0 - 8)) {  // Clip right
    const v128 d_shuff = v128_from_v64(v64_from_64(0x0f0f0e0d0c0b0a09LL),
                                       v64_from_64(0x0707060504030201LL));
    const v128 e_shuff = v128_from_v64(v64_from_64(0x0f0f0f0e0d0c0b0aLL),
                                       v64_from_64(0x0707070605040302LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v64 k1 = v64_load_aligned(org);
      const v64 k2 = v64_load_aligned(org + ostride);
      const v64 l1 = v64_load_aligned(rec);
      const v64 l2 = v64_load_aligned(rec + rstride);
      v128 o = v128_from_v64(k1, k2);
      const v128 q = v128_from_v64(l1, l2);
      const v128 x = v128_add_8(c128, q);
      v128 a = v128_add_8(
          c128,
          v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1));
      v128 b = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec - 2),
                                        v64_load_unaligned(rec - 2 + rstride)));
      v128 c = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec - 1),
                                        v64_load_unaligned(rec - 1 + rstride)));
      v128 d = v128_shuffle_8(x, d_shuff);
      v128 e = v128_shuffle_8(x, e_shuff);
      v128 f = v128_add_8(
          c128, v128_from_v64(
                    l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      calc_delta_multi(x, q, o, a, b, c, d, e, f, cp1, cm1, cp2, cm2, cp4, cm4,
                       &ssd1, &ssd2, &ssd3);
      rec += 2 * rstride;
      org += 2 * ostride;
    }
  } else {  // No left/right clipping
    int y;
    for (y = 0; y < 8; y += 2) {
      const v64 k1 = v64_load_aligned(org);
      const v64 k2 = v64_load_aligned(org + ostride);
      const v64 l1 = v64_load_aligned(rec);
      const v64 l2 = v64_load_aligned(rec + rstride);
      v128 o = v128_from_v64(k1, k2);
      const v128 q = v128_from_v64(l1, l2);
      const v128 x = v128_add_8(c128, q);
      v128 a = v128_add_8(
          c128,
          v128_from_v64(v64_load_aligned(rec - (y != -y0) * rstride), l1));
      v128 b = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec - 2),
                                        v64_load_unaligned(rec - 2 + rstride)));
      v128 c = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec - 1),
                                        v64_load_unaligned(rec - 1 + rstride)));
      v128 d = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec + 1),
                                        v64_load_unaligned(rec + 1 + rstride)));
      v128 e = v128_add_8(c128,
                          v128_from_v64(v64_load_unaligned(rec + 2),
                                        v64_load_unaligned(rec + 2 + rstride)));
      v128 f = v128_add_8(
          c128, v128_from_v64(
                    l2, v64_load_aligned(rec + ((y != bottom) + 1) * rstride)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      calc_delta_multi(x, q, o, a, b, c, d, e, f, cp1, cm1, cp2, cm2, cp4, cm4,
                       &ssd1, &ssd2, &ssd3);
      rec += 2 * rstride;
      org += 2 * ostride;
    }
  }
  sum[0] += v128_ssd_u8_sum(ssd0);
  sum[1] += v128_ssd_u8_sum(ssd1);
  sum[2] += v128_ssd_u8_sum(ssd2);
  sum[3] += v128_ssd_u8_sum(ssd3);
}

#if CONFIG_AOM_HIGHBITDEPTH
void SIMD_FUNC(aom_clpf_detect_hbd)(const uint16_t *rec, const uint16_t *org,
                                    int rstride, int ostride, int x0, int y0,
                                    int width, int height, int *sum0, int *sum1,
                                    unsigned int strength, int shift) {
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  const v128 c128 = v128_dup_8(128);
  const v128 sp = v128_dup_8(strength >> shift);
  const v128 sm = v128_dup_8(-(int)(strength >> shift));
  const int bottom = height - 2 - y0;

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  if (!x0) {  // Clip left
    const v128 b_shuff = v128_from_v64(v64_from_64(0x0d0c0b0a09080808LL),
                                       v64_from_64(0x0504030201000000LL));
    const v128 c_shuff = v128_from_v64(v64_from_64(0x0e0d0c0b0a090808LL),
                                       v64_from_64(0x0605040302010000LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
      const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
      const v128 o =
          v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                         v128_shr_u16(v128_load_aligned(org + ostride), shift));
      const v128 q = v128_unziplo_8(n1, n2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128, v128_unziplo_8(
                    v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride),
                                 shift),
                    n1));
      const v128 b = v128_shuffle_8(x, b_shuff);
      const v128 c = v128_shuffle_8(x, c_shuff);
      const v128 d = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 1), shift),
              v128_shr_u16(v128_load_unaligned(rec + 1 + rstride), shift)));
      const v128 e = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 2), shift),
              v128_shr_u16(v128_load_unaligned(rec + 2 + rstride), shift)));
      const v128 f = v128_add_8(
          c128, v128_unziplo_8(
                    n2, v128_shr_u16(v128_load_unaligned(
                                         rec + ((y != bottom) + 1) * rstride),
                                     shift)));

      ssd0 = v128_ssd_u8(ssd0, o, q);
      ssd1 = v128_ssd_u8(
          ssd1, o, v128_add_8(q, calc_delta(x, a, b, c, d, e, f, sp, sm)));
      rec += rstride * 2;
      org += ostride * 2;
    }
  } else if (!(width - x0 - 8)) {  // Clip right
    const v128 d_shuff = v128_from_v64(v64_from_64(0x0f0f0e0d0c0b0a09LL),
                                       v64_from_64(0x0707060504030201LL));
    const v128 e_shuff = v128_from_v64(v64_from_64(0x0f0f0f0e0d0c0b0aLL),
                                       v64_from_64(0x0707070605040302LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
      const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
      const v128 o =
          v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                         v128_shr_u16(v128_load_aligned(org + ostride), shift));
      const v128 q = v128_unziplo_8(n1, n2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128, v128_unziplo_8(
                    v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride),
                                 shift),
                    n1));
      const v128 b = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 2), shift),
              v128_shr_u16(v128_load_unaligned(rec - 2 + rstride), shift)));
      const v128 c = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 1), shift),
              v128_shr_u16(v128_load_unaligned(rec - 1 + rstride), shift)));
      const v128 d = v128_shuffle_8(x, d_shuff);
      const v128 e = v128_shuffle_8(x, e_shuff);
      const v128 f = v128_add_8(
          c128, v128_unziplo_8(
                    n2, v128_shr_u16(v128_load_unaligned(
                                         rec + ((y != bottom) + 1) * rstride),
                                     shift)));

      ssd0 = v128_ssd_u8(ssd0, o, q);
      ssd1 = v128_ssd_u8(
          ssd1, o, v128_add_8(q, calc_delta(x, a, b, c, d, e, f, sp, sm)));
      rec += rstride * 2;
      org += ostride * 2;
    }
  } else {  // No left/right clipping
    int y;
    for (y = 0; y < 8; y += 2) {
      const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
      const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
      const v128 o =
          v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                         v128_shr_u16(v128_load_aligned(org + ostride), shift));
      const v128 q = v128_unziplo_8(n1, n2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128, v128_unziplo_8(
                    v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride),
                                 shift),
                    n1));
      const v128 b = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 2), shift),
              v128_shr_u16(v128_load_unaligned(rec - 2 + rstride), shift)));
      const v128 c = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 1), shift),
              v128_shr_u16(v128_load_unaligned(rec - 1 + rstride), shift)));
      const v128 d = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 1), shift),
              v128_shr_u16(v128_load_unaligned(rec + 1 + rstride), shift)));
      const v128 e = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 2), shift),
              v128_shr_u16(v128_load_unaligned(rec + 2 + rstride), shift)));
      const v128 f = v128_add_8(
          c128, v128_unziplo_8(
                    n2, v128_shr_u16(v128_load_unaligned(
                                         rec + ((y != bottom) + 1) * rstride),
                                     shift)));
      ssd0 = v128_ssd_u8(ssd0, o, q);
      ssd1 = v128_ssd_u8(
          ssd1, o, v128_add_8(q, calc_delta(x, a, b, c, d, e, f, sp, sm)));
      rec += rstride * 2;
      org += ostride * 2;
    }
  }
  *sum0 += v128_ssd_u8_sum(ssd0);
  *sum1 += v128_ssd_u8_sum(ssd1);
}

void SIMD_FUNC(aom_clpf_detect_multi_hbd)(const uint16_t *rec,
                                          const uint16_t *org, int rstride,
                                          int ostride, int x0, int y0,
                                          int width, int height, int *sum,
                                          int shift) {
  const v128 c128 = v128_dup_8(128);
  const v128 cp1 = v128_dup_8(1);
  const v128 cm1 = v128_dup_8(-1);
  const v128 cp2 = v128_dup_8(2);
  const v128 cm2 = v128_dup_8(-2);
  const v128 cp4 = v128_dup_8(4);
  const v128 cm4 = v128_dup_8(-4);
  const int bottom = height - 2 - y0;
  ssd128_internal ssd0 = v128_ssd_u8_init();
  ssd128_internal ssd1 = v128_ssd_u8_init();
  ssd128_internal ssd2 = v128_ssd_u8_init();
  ssd128_internal ssd3 = v128_ssd_u8_init();

  rec += x0 + y0 * rstride;
  org += x0 + y0 * ostride;

  if (!x0) {  // Clip left
    const v128 b_shuff = v128_from_v64(v64_from_64(0x0d0c0b0a09080808LL),
                                       v64_from_64(0x0504030201000000LL));
    const v128 c_shuff = v128_from_v64(v64_from_64(0x0e0d0c0b0a090808LL),
                                       v64_from_64(0x0605040302010000LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
      const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
      const v128 o =
          v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                         v128_shr_u16(v128_load_aligned(org + ostride), shift));
      const v128 q = v128_unziplo_8(n1, n2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128, v128_unziplo_8(
                    v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride),
                                 shift),
                    n1));
      const v128 b = v128_shuffle_8(x, b_shuff);
      const v128 c = v128_shuffle_8(x, c_shuff);
      const v128 d = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 1), shift),
              v128_shr_u16(v128_load_unaligned(rec + 1 + rstride), shift)));
      const v128 e = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 2), shift),
              v128_shr_u16(v128_load_unaligned(rec + 2 + rstride), shift)));
      const v128 f = v128_add_8(
          c128, v128_unziplo_8(
                    n2, v128_shr_u16(v128_load_unaligned(
                                         rec + ((y != bottom) + 1) * rstride),
                                     shift)));

      ssd0 = v128_ssd_u8(ssd0, o, q);
      calc_delta_multi(x, q, o, a, b, c, d, e, f, cp1, cm1, cp2, cm2, cp4, cm4,
                       &ssd1, &ssd2, &ssd3);
      rec += 2 * rstride;
      org += 2 * ostride;
    }
  } else if (!(width - x0 - 8)) {  // Clip right
    const v128 d_shuff = v128_from_v64(v64_from_64(0x0f0f0e0d0c0b0a09LL),
                                       v64_from_64(0x0707060504030201LL));
    const v128 e_shuff = v128_from_v64(v64_from_64(0x0f0f0f0e0d0c0b0aLL),
                                       v64_from_64(0x0707070605040302LL));
    int y;

    for (y = 0; y < 8; y += 2) {
      const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
      const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
      const v128 o =
          v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                         v128_shr_u16(v128_load_aligned(org + ostride), shift));
      const v128 q = v128_unziplo_8(n1, n2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128, v128_unziplo_8(
                    v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride),
                                 shift),
                    n1));
      const v128 b = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 2), shift),
              v128_shr_u16(v128_load_unaligned(rec - 2 + rstride), shift)));
      const v128 c = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 1), shift),
              v128_shr_u16(v128_load_unaligned(rec - 1 + rstride), shift)));
      const v128 d = v128_shuffle_8(x, d_shuff);
      const v128 e = v128_shuffle_8(x, e_shuff);
      const v128 f = v128_add_8(
          c128, v128_unziplo_8(
                    n2, v128_shr_u16(v128_load_unaligned(
                                         rec + ((y != bottom) + 1) * rstride),
                                     shift)));

      ssd0 = v128_ssd_u8(ssd0, o, q);
      calc_delta_multi(x, q, o, a, b, c, d, e, f, cp1, cm1, cp2, cm2, cp4, cm4,
                       &ssd1, &ssd2, &ssd3);
      rec += 2 * rstride;
      org += 2 * ostride;
    }
  } else {  // No left/right clipping
    int y;
    for (y = 0; y < 8; y += 2) {
      const v128 n1 = v128_shr_u16(v128_load_aligned(rec), shift);
      const v128 n2 = v128_shr_u16(v128_load_aligned(rec + rstride), shift);
      const v128 o =
          v128_unziplo_8(v128_shr_u16(v128_load_aligned(org), shift),
                         v128_shr_u16(v128_load_aligned(org + ostride), shift));
      const v128 q = v128_unziplo_8(n1, n2);
      const v128 x = v128_add_8(c128, q);
      const v128 a = v128_add_8(
          c128, v128_unziplo_8(
                    v128_shr_u16(v128_load_aligned(rec - (y != -y0) * rstride),
                                 shift),
                    n1));
      const v128 b = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 2), shift),
              v128_shr_u16(v128_load_unaligned(rec - 2 + rstride), shift)));
      const v128 c = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec - 1), shift),
              v128_shr_u16(v128_load_unaligned(rec - 1 + rstride), shift)));
      const v128 d = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 1), shift),
              v128_shr_u16(v128_load_unaligned(rec + 1 + rstride), shift)));
      const v128 e = v128_add_8(
          c128,
          v128_unziplo_8(
              v128_shr_u16(v128_load_unaligned(rec + 2), shift),
              v128_shr_u16(v128_load_unaligned(rec + 2 + rstride), shift)));
      const v128 f = v128_add_8(
          c128, v128_unziplo_8(
                    n2, v128_shr_u16(v128_load_unaligned(
                                         rec + ((y != bottom) + 1) * rstride),
                                     shift)));

      ssd0 = v128_ssd_u8(ssd0, o, q);
      calc_delta_multi(x, q, o, a, b, c, d, e, f, cp1, cm1, cp2, cm2, cp4, cm4,
                       &ssd1, &ssd2, &ssd3);
      rec += 2 * rstride;
      org += 2 * ostride;
    }
  }
  sum[0] += v128_ssd_u8_sum(ssd0);
  sum[1] += v128_ssd_u8_sum(ssd1);
  sum[2] += v128_ssd_u8_sum(ssd2);
  sum[3] += v128_ssd_u8_sum(ssd3);
}
#endif