aom_dsp/x86/highbd_intrapred_avx2.c

/*
 * Copyright (c) 2017, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <immintrin.h>

#include "aom_ports/msvc.h"
#include "./aom_dsp_rtcd.h"

// -----------------------------------------------------------------------------
// D45E_PRED
/*
; ------------------------------------------
; input: x, y, z, result
;
; trick from pascal
; (x+2y+z+2)>>2 can be calculated as:
; result = avg(x,z)
; result -= xor(x,z) & 1
; result = avg(result,y)
; ------------------------------------------
*/
static INLINE __m256i avg3_epu16(const __m256i *x, const __m256i *y,
                                 const __m256i *z) {
  const __m256i one = _mm256_set1_epi16(1);
  const __m256i a = _mm256_avg_epu16(*x, *z);
  const __m256i b =
      _mm256_subs_epu16(a, _mm256_and_si256(_mm256_xor_si256(*x, *z), one));
  return _mm256_avg_epu16(b, *y);
}

static INLINE void d45e_w16(const __m256i *a0, const __m256i *a1,
                            const __m256i *a2, uint16_t **dst,
                            ptrdiff_t stride) {
  const __m256i y = avg3_epu16(a0, a1, a2);
  _mm256_storeu_si256((__m256i *)*dst, y);
  *dst += stride;
}

void aom_highbd_d45e_predictor_16x8_avx2(uint16_t *dst, ptrdiff_t stride,
                                         const uint16_t *above,
                                         const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  __m256i x0 = _mm256_loadu_si256((const __m256i *)above);
  __m256i x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  __m256i x2 = _mm256_loadu_si256((const __m256i *)(above + 2));

  d45e_w16(&x0, &x1, &x2, &dst, stride);

  int i = 3;
  do {
    x0 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x1, &x2, &x0, &dst, stride);

    x1 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x2, &x0, &x1, &dst, stride);

    x2 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x0, &x1, &x2, &dst, stride);
  } while (i < 9);

  x0 = _mm256_loadu_si256((const __m256i *)(above + 9));
  x0 = _mm256_insert_epi16(x0, above[23], 15);
  const __m256i y = avg3_epu16(&x1, &x2, &x0);
  _mm256_storeu_si256((__m256i *)dst, y);
}

void aom_highbd_d45e_predictor_16x16_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  __m256i x0 = _mm256_loadu_si256((const __m256i *)above);
  __m256i x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  __m256i x2 = _mm256_loadu_si256((const __m256i *)(above + 2));

  d45e_w16(&x0, &x1, &x2, &dst, stride);

  int i = 3;
  do {
    x0 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x1, &x2, &x0, &dst, stride);

    x1 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x2, &x0, &x1, &dst, stride);

    x2 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x0, &x1, &x2, &dst, stride);
  } while (i < 15);

  x0 = _mm256_loadu_si256((const __m256i *)(above + 15));
  d45e_w16(&x1, &x2, &x0, &dst, stride);

  x1 = _mm256_loadu_si256((const __m256i *)(above + 16));
  d45e_w16(&x2, &x0, &x1, &dst, stride);

  x2 = _mm256_loadu_si256((const __m256i *)(above + 17));
  x2 = _mm256_insert_epi16(x2, above[31], 15);
  const __m256i y = avg3_epu16(&x0, &x1, &x2);
  _mm256_storeu_si256((__m256i *)dst, y);
}

void aom_highbd_d45e_predictor_16x32_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  __m256i x0 = _mm256_loadu_si256((const __m256i *)above);
  __m256i x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  __m256i x2 = _mm256_loadu_si256((const __m256i *)(above + 2));

  d45e_w16(&x0, &x1, &x2, &dst, stride);

  int i = 3;
  do {
    x0 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x1, &x2, &x0, &dst, stride);

    x1 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x2, &x0, &x1, &dst, stride);

    x2 = _mm256_loadu_si256((const __m256i *)(above + i++));
    d45e_w16(&x0, &x1, &x2, &dst, stride);
  } while (i < 33);

  x0 = _mm256_loadu_si256((const __m256i *)(above + 33));
  x0 = _mm256_insert_epi16(x0, above[47], 15);
  const __m256i y = avg3_epu16(&x1, &x2, &x0);
  _mm256_storeu_si256((__m256i *)dst, y);
}

void aom_highbd_d45e_predictor_32x16_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  __m256i x0 = _mm256_loadu_si256((const __m256i *)above);
  __m256i x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  __m256i x2 = _mm256_loadu_si256((const __m256i *)(above + 2));
  __m256i y0 = _mm256_loadu_si256((const __m256i *)(above + 16));
  __m256i y1 = _mm256_loadu_si256((const __m256i *)(above + 17));
  __m256i y2 = _mm256_loadu_si256((const __m256i *)(above + 18));

  uint16_t *dst1 = dst;
  uint16_t *dst2 = dst + 16;

  d45e_w16(&x0, &x1, &x2, &dst1, stride);
  d45e_w16(&y0, &y1, &y2, &dst2, stride);

  int i = 3;
  do {
    x0 = _mm256_loadu_si256((const __m256i *)(above + i));
    d45e_w16(&x1, &x2, &x0, &dst1, stride);
    y0 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    d45e_w16(&y1, &y2, &y0, &dst2, stride);

    x1 = _mm256_loadu_si256((const __m256i *)(above + i));
    d45e_w16(&x2, &x0, &x1, &dst1, stride);
    y1 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    d45e_w16(&y2, &y0, &y1, &dst2, stride);

    x2 = _mm256_loadu_si256((const __m256i *)(above + i));
    d45e_w16(&x0, &x1, &x2, &dst1, stride);
    y2 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    d45e_w16(&y0, &y1, &y2, &dst2, stride);
  } while (i < 15);

  x0 = _mm256_loadu_si256((const __m256i *)(above + 15));
  d45e_w16(&x1, &x2, &x0, &dst1, stride);
  y0 = _mm256_loadu_si256((const __m256i *)(above + 16 + 15));
  d45e_w16(&y1, &y2, &y0, &dst2, stride);

  x1 = _mm256_loadu_si256((const __m256i *)(above + 16));
  d45e_w16(&x2, &x0, &x1, &dst1, stride);
  y1 = _mm256_loadu_si256((const __m256i *)(above + 16 + 16));
  d45e_w16(&y2, &y0, &y1, &dst2, stride);

  x2 = _mm256_loadu_si256((const __m256i *)(above + 17));
  __m256i u = avg3_epu16(&x0, &x1, &x2);
  _mm256_storeu_si256((__m256i *)dst1, u);

  y2 = _mm256_loadu_si256((const __m256i *)(above + 16 + 17));
  y2 = _mm256_insert_epi16(y2, above[47], 15);
  u = avg3_epu16(&y0, &y1, &y2);
  _mm256_storeu_si256((__m256i *)dst2, u);
}

void aom_highbd_d45e_predictor_32x32_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  __m256i x0 = _mm256_loadu_si256((const __m256i *)above);
  __m256i x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  __m256i x2 = _mm256_loadu_si256((const __m256i *)(above + 2));
  __m256i y0 = _mm256_loadu_si256((const __m256i *)(above + 16));
  __m256i y1 = _mm256_loadu_si256((const __m256i *)(above + 17));
  __m256i y2 = _mm256_loadu_si256((const __m256i *)(above + 18));

  uint16_t *dst1 = dst;
  uint16_t *dst2 = dst + 16;

  d45e_w16(&x0, &x1, &x2, &dst1, stride);
  d45e_w16(&y0, &y1, &y2, &dst2, stride);

  int i = 3;
  do {
    x0 = _mm256_loadu_si256((const __m256i *)(above + i));
    d45e_w16(&x1, &x2, &x0, &dst1, stride);
    y0 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    d45e_w16(&y1, &y2, &y0, &dst2, stride);

    x1 = _mm256_loadu_si256((const __m256i *)(above + i));
    d45e_w16(&x2, &x0, &x1, &dst1, stride);
    y1 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    d45e_w16(&y2, &y0, &y1, &dst2, stride);

    x2 = _mm256_loadu_si256((const __m256i *)(above + i));
    d45e_w16(&x0, &x1, &x2, &dst1, stride);
    y2 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    d45e_w16(&y0, &y1, &y2, &dst2, stride);
  } while (i < 33);

  x0 = _mm256_loadu_si256((const __m256i *)(above + 33));
  __m256i u = avg3_epu16(&x1, &x2, &x0);
  _mm256_storeu_si256((__m256i *)dst1, u);

  y0 = _mm256_loadu_si256((const __m256i *)(above + 16 + 33));
  y0 = _mm256_insert_epi16(y0, above[63], 15);
  u = avg3_epu16(&y1, &y2, &y0);
  _mm256_storeu_si256((__m256i *)dst2, u);
}

// -----------------------------------------------------------------------------
// D207E_PRED

static INLINE void d207_32x4(const uint16_t *left, uint16_t **dst,
                             ptrdiff_t stride) {
  const __m256i x0 = _mm256_loadu_si256((const __m256i *)left);
  const __m256i x1 = _mm256_loadu_si256((const __m256i *)(left + 1));
  const __m256i x2 = _mm256_loadu_si256((const __m256i *)(left + 2));
  const __m256i x3 = _mm256_loadu_si256((const __m256i *)(left + 3));
  const __m256i x4 = _mm256_loadu_si256((const __m256i *)(left + 4));
  const __m256i x5 = _mm256_loadu_si256((const __m256i *)(left + 5));

  const __m256i y0 = _mm256_avg_epu16(x0, x1);
  const __m256i y1 = _mm256_avg_epu16(x1, x2);
  const __m256i y2 = _mm256_avg_epu16(x2, x3);
  const __m256i y3 = _mm256_avg_epu16(x3, x4);

  const __m256i u0 = avg3_epu16(&x0, &x1, &x2);
  const __m256i u1 = avg3_epu16(&x1, &x2, &x3);
  const __m256i u2 = avg3_epu16(&x2, &x3, &x4);
  const __m256i u3 = avg3_epu16(&x3, &x4, &x5);

  __m256i v0 = _mm256_unpacklo_epi16(y0, u0);
  __m256i v1 = _mm256_unpackhi_epi16(y0, u0);
  _mm256_storeu_si256((__m256i *)*dst, _mm256_permute2x128_si256(v0, v1, 0x20));
  _mm256_storeu_si256((__m256i *)(*dst + 16),
                      _mm256_permute2x128_si256(v0, v1, 0x31));
  *dst += stride;

  v0 = _mm256_unpacklo_epi16(y1, u1);
  v1 = _mm256_unpackhi_epi16(y1, u1);
  _mm256_storeu_si256((__m256i *)*dst, _mm256_permute2x128_si256(v0, v1, 0x20));
  _mm256_storeu_si256((__m256i *)(*dst + 16),
                      _mm256_permute2x128_si256(v0, v1, 0x31));
  *dst += stride;

  v0 = _mm256_unpacklo_epi16(y2, u2);
  v1 = _mm256_unpackhi_epi16(y2, u2);
  _mm256_storeu_si256((__m256i *)*dst, _mm256_permute2x128_si256(v0, v1, 0x20));
  _mm256_storeu_si256((__m256i *)(*dst + 16),
                      _mm256_permute2x128_si256(v0, v1, 0x31));
  *dst += stride;

  v0 = _mm256_unpacklo_epi16(y3, u3);
  v1 = _mm256_unpackhi_epi16(y3, u3);
  _mm256_storeu_si256((__m256i *)*dst, _mm256_permute2x128_si256(v0, v1, 0x20));
  _mm256_storeu_si256((__m256i *)(*dst + 16),
                      _mm256_permute2x128_si256(v0, v1, 0x31));
  *dst += stride;
}

void aom_highbd_d207e_predictor_32x16_avx2(uint16_t *dst, ptrdiff_t stride,
                                           const uint16_t *above,
                                           const uint16_t *left, int bd) {
  (void)above;
  (void)bd;
  int i;
  for (i = 0; i < 16; i += 4) {
    d207_32x4(left + i, &dst, stride);
  }
}

void aom_highbd_d207e_predictor_32x32_avx2(uint16_t *dst, ptrdiff_t stride,
                                           const uint16_t *above,
                                           const uint16_t *left, int bd) {
  (void)above;
  (void)bd;
  int i;
  for (i = 0; i < 32; i += 4) {
    d207_32x4(left + i, &dst, stride);
  }
}

#define D63E_STORE_16X4                      \
  do {                                       \
    _mm256_storeu_si256((__m256i *)dst, y0); \
    dst += stride;                           \
    _mm256_storeu_si256((__m256i *)dst, u0); \
    dst += stride;                           \
    _mm256_storeu_si256((__m256i *)dst, y1); \
    dst += stride;                           \
    _mm256_storeu_si256((__m256i *)dst, u1); \
    dst += stride;                           \
  } while (0)

void aom_highbd_d63e_predictor_16x8_avx2(uint16_t *dst, ptrdiff_t stride,
                                         const uint16_t *above,
                                         const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  __m256i x0 = _mm256_loadu_si256((const __m256i *)above);
  __m256i x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  const __m256i x2 = _mm256_loadu_si256((const __m256i *)(above + 2));
  const __m256i x3 = _mm256_loadu_si256((const __m256i *)(above + 3));

  __m256i y0 = _mm256_avg_epu16(x0, x1);
  __m256i y1 = _mm256_avg_epu16(x1, x2);

  __m256i u0 = avg3_epu16(&x0, &x1, &x2);
  __m256i u1 = avg3_epu16(&x1, &x2, &x3);

  D63E_STORE_16X4;

  x0 = _mm256_loadu_si256((const __m256i *)(above + 4));
  x1 = _mm256_loadu_si256((const __m256i *)(above + 5));

  y0 = _mm256_avg_epu16(x2, x3);
  y1 = _mm256_avg_epu16(x3, x0);

  u0 = avg3_epu16(&x2, &x3, &x0);
  u1 = avg3_epu16(&x3, &x0, &x1);

  D63E_STORE_16X4;
}

static INLINE void d63e_w16(const uint16_t *above, uint16_t *dst,
                            ptrdiff_t stride, int num) {
  __m256i x0, x1, x2, x3;
  __m256i y0, y1, u0, u1;
  const int count = (num >> 1) + 2;

  x0 = _mm256_loadu_si256((const __m256i *)above);
  x1 = _mm256_loadu_si256((const __m256i *)(above + 1));

  int i = 2;
  do {
    x2 = _mm256_loadu_si256((const __m256i *)(above + i++));
    x3 = _mm256_loadu_si256((const __m256i *)(above + i++));

    y0 = _mm256_avg_epu16(x0, x1);
    y1 = _mm256_avg_epu16(x1, x2);

    u0 = avg3_epu16(&x0, &x1, &x2);
    u1 = avg3_epu16(&x1, &x2, &x3);

    D63E_STORE_16X4;

    x0 = _mm256_loadu_si256((const __m256i *)(above + i++));
    x1 = _mm256_loadu_si256((const __m256i *)(above + i++));

    y0 = _mm256_avg_epu16(x2, x3);
    y1 = _mm256_avg_epu16(x3, x0);

    u0 = avg3_epu16(&x2, &x3, &x0);
    u1 = avg3_epu16(&x3, &x0, &x1);

    D63E_STORE_16X4;
  } while (i < count);
}

void aom_highbd_d63e_predictor_16x16_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  d63e_w16(above, dst, stride, 16);
}

void aom_highbd_d63e_predictor_16x32_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  d63e_w16(above, dst, stride, 32);
}

#define D63E_STORE_32X4                             \
  do {                                              \
    _mm256_storeu_si256((__m256i *)dst, y0);        \
    _mm256_storeu_si256((__m256i *)(dst + 16), z0); \
    dst += stride;                                  \
    _mm256_storeu_si256((__m256i *)dst, u0);        \
    _mm256_storeu_si256((__m256i *)(dst + 16), v0); \
    dst += stride;                                  \
    _mm256_storeu_si256((__m256i *)dst, y1);        \
    _mm256_storeu_si256((__m256i *)(dst + 16), z1); \
    dst += stride;                                  \
    _mm256_storeu_si256((__m256i *)dst, u1);        \
    _mm256_storeu_si256((__m256i *)(dst + 16), v1); \
    dst += stride;                                  \
  } while (0)

static INLINE void d63e_w32(const uint16_t *above, uint16_t *dst,
                            ptrdiff_t stride, int num) {
  __m256i x0, x1, x2, x3, a0, a1, a2, a3;
  __m256i y0, y1, u0, u1, z0, z1, v0, v1;
  const int count = (num >> 1) + 2;

  x0 = _mm256_loadu_si256((const __m256i *)above);
  x1 = _mm256_loadu_si256((const __m256i *)(above + 1));
  a0 = _mm256_loadu_si256((const __m256i *)(above + 16));
  a1 = _mm256_loadu_si256((const __m256i *)(above + 16 + 1));

  int i = 2;
  do {
    x2 = _mm256_loadu_si256((const __m256i *)(above + i));
    a2 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    x3 = _mm256_loadu_si256((const __m256i *)(above + i));
    a3 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));

    y0 = _mm256_avg_epu16(x0, x1);
    y1 = _mm256_avg_epu16(x1, x2);

    u0 = avg3_epu16(&x0, &x1, &x2);
    u1 = avg3_epu16(&x1, &x2, &x3);

    z0 = _mm256_avg_epu16(a0, a1);
    z1 = _mm256_avg_epu16(a1, a2);

    v0 = avg3_epu16(&a0, &a1, &a2);
    v1 = avg3_epu16(&a1, &a2, &a3);

    D63E_STORE_32X4;

    x0 = _mm256_loadu_si256((const __m256i *)(above + i));
    a0 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));
    x1 = _mm256_loadu_si256((const __m256i *)(above + i));
    a1 = _mm256_loadu_si256((const __m256i *)(above + 16 + i++));

    y0 = _mm256_avg_epu16(x2, x3);
    y1 = _mm256_avg_epu16(x3, x0);

    u0 = avg3_epu16(&x2, &x3, &x0);
    u1 = avg3_epu16(&x3, &x0, &x1);

    z0 = _mm256_avg_epu16(a2, a3);
    z1 = _mm256_avg_epu16(a3, a0);

    v0 = avg3_epu16(&a2, &a3, &a0);
    v1 = avg3_epu16(&a3, &a0, &a1);

    D63E_STORE_32X4;
  } while (i < count);
}

void aom_highbd_d63e_predictor_32x16_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  d63e_w32(above, dst, stride, 16);
}

void aom_highbd_d63e_predictor_32x32_avx2(uint16_t *dst, ptrdiff_t stride,
                                          const uint16_t *above,
                                          const uint16_t *left, int bd) {
  (void)left;
  (void)bd;
  d63e_w32(above, dst, stride, 32);
}