/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <smmintrin.h>

#include "./av1_rtcd.h"
#include "aom_ports/mem.h"
#include "av1/common/enums.h"
#include "av1/common/reconintra.h"

static INLINE void AddPixelsSmall(const uint8_t *above, const uint8_t *left,
                                  __m128i *sum) {
  const __m128i a = _mm_loadu_si128((const __m128i *)above);
  const __m128i l = _mm_loadu_si128((const __m128i *)left);
  const __m128i zero = _mm_setzero_si128();

  __m128i u0 = _mm_unpacklo_epi8(a, zero);
  __m128i u1 = _mm_unpacklo_epi8(l, zero);

  sum[0] = _mm_add_epi16(u0, u1);
}

static INLINE int GetMeanValue4x4(const uint8_t *above, const uint8_t *left,
                                  __m128i *params) {
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector, u;
  uint16_t sum_value;

  AddPixelsSmall(above, left, &sum_vector);

  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 2 values
  u = _mm_srli_si128(sum_vector, 2);
  sum_vector = _mm_add_epi16(sum_vector, u);

  sum_value = _mm_extract_epi16(sum_vector, 0);
  sum_value += 4;
  sum_value >>= 3;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

static INLINE int GetMeanValue8x8(const uint8_t *above, const uint8_t *left,
                                  __m128i *params) {
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector, u;
  uint16_t sum_value;

  AddPixelsSmall(above, left, &sum_vector);

  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 4 values
  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 2 values

  u = _mm_srli_si128(sum_vector, 2);
  sum_vector = _mm_add_epi16(sum_vector, u);

  sum_value = _mm_extract_epi16(sum_vector, 0);
  sum_value += 8;
  sum_value >>= 4;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

static INLINE void AddPixelsLarge(const uint8_t *above, const uint8_t *left,
                                  __m128i *sum) {
  const __m128i a = _mm_loadu_si128((const __m128i *)above);
  const __m128i l = _mm_loadu_si128((const __m128i *)left);
  const __m128i zero = _mm_setzero_si128();

  __m128i u0 = _mm_unpacklo_epi8(a, zero);
  __m128i u1 = _mm_unpacklo_epi8(l, zero);

  sum[0] = _mm_add_epi16(u0, u1);

  u0 = _mm_unpackhi_epi8(a, zero);
  u1 = _mm_unpackhi_epi8(l, zero);

  sum[0] = _mm_add_epi16(sum[0], u0);
  sum[0] = _mm_add_epi16(sum[0], u1);
}

static INLINE int GetMeanValue16x16(const uint8_t *above, const uint8_t *left,
                                    __m128i *params) {
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector, u;
  uint16_t sum_value;

  AddPixelsLarge(above, left, &sum_vector);

  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 4 values
  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 2 values

  u = _mm_srli_si128(sum_vector, 2);
  sum_vector = _mm_add_epi16(sum_vector, u);

  sum_value = _mm_extract_epi16(sum_vector, 0);
  sum_value += 16;
  sum_value >>= 5;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

static INLINE int GetMeanValue32x32(const uint8_t *above, const uint8_t *left,
                                    __m128i *params) {
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector[2], u;
  uint16_t sum_value;

  AddPixelsLarge(above, left, &sum_vector[0]);
  AddPixelsLarge(above + 16, left + 16, &sum_vector[1]);

  sum_vector[0] = _mm_add_epi16(sum_vector[0], sum_vector[1]);
  sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero);  // still has 4 values
  sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero);  // still has 2 values

  u = _mm_srli_si128(sum_vector[0], 2);
  sum_vector[0] = _mm_add_epi16(sum_vector[0], u);

  sum_value = _mm_extract_epi16(sum_vector[0], 0);
  sum_value += 32;
  sum_value >>= 6;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

// Note:
//  params[4] : mean value, 4 int32_t repetition
//
static INLINE int CalcRefPixelsMeanValue(const uint8_t *above,
                                         const uint8_t *left, int bs,
                                         __m128i *params) {
  int meanValue = 0;
  switch (bs) {
    case 4: meanValue = GetMeanValue4x4(above, left, params); break;
    case 8: meanValue = GetMeanValue8x8(above, left, params); break;
    case 16: meanValue = GetMeanValue16x16(above, left, params); break;
    case 32: meanValue = GetMeanValue32x32(above, left, params); break;
    default: assert(0);
  }
  return meanValue;
}

// Note:
//  params[0-3] : 4-tap filter coefficients (int32_t per coefficient)
//
static INLINE void GetIntraFilterParams(int bs, int mode, __m128i *params) {
  const TX_SIZE tx_size =
      (bs == 32) ? TX_32X32
                 : ((bs == 16) ? TX_16X16 : ((bs == 8) ? TX_8X8 : (TX_4X4)));
  // c0
  params[0] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][0],
                            av1_filter_intra_taps_4[tx_size][mode][0],
                            av1_filter_intra_taps_4[tx_size][mode][0],
                            av1_filter_intra_taps_4[tx_size][mode][0]);
  // c1
  params[1] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][1],
                            av1_filter_intra_taps_4[tx_size][mode][1],
                            av1_filter_intra_taps_4[tx_size][mode][1],
                            av1_filter_intra_taps_4[tx_size][mode][1]);
  // c2
  params[2] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][2],
                            av1_filter_intra_taps_4[tx_size][mode][2],
                            av1_filter_intra_taps_4[tx_size][mode][2],
                            av1_filter_intra_taps_4[tx_size][mode][2]);
  // c3
  params[3] = _mm_set_epi32(av1_filter_intra_taps_4[tx_size][mode][3],
                            av1_filter_intra_taps_4[tx_size][mode][3],
                            av1_filter_intra_taps_4[tx_size][mode][3],
                            av1_filter_intra_taps_4[tx_size][mode][3]);
}

static const int maxBlkSize = 32;

static INLINE void SavePred4x4(int *pred, const __m128i *mean, uint8_t *dst,
                               ptrdiff_t stride) {
  const int predStride = (maxBlkSize << 1) + 1;
  __m128i p0 = _mm_loadu_si128((const __m128i *)pred);
  __m128i p1 = _mm_loadu_si128((const __m128i *)(pred + predStride));
  __m128i p2 = _mm_loadu_si128((const __m128i *)(pred + 2 * predStride));
  __m128i p3 = _mm_loadu_si128((const __m128i *)(pred + 3 * predStride));

  p0 = _mm_add_epi32(p0, mean[0]);
  p1 = _mm_add_epi32(p1, mean[0]);
  p2 = _mm_add_epi32(p2, mean[0]);
  p3 = _mm_add_epi32(p3, mean[0]);

  p0 = _mm_packus_epi32(p0, p1);
  p1 = _mm_packus_epi32(p2, p3);
  p0 = _mm_packus_epi16(p0, p1);

  *((int *)dst) = _mm_cvtsi128_si32(p0);
  p0 = _mm_srli_si128(p0, 4);
  *((int *)(dst + stride)) = _mm_cvtsi128_si32(p0);
  p0 = _mm_srli_si128(p0, 4);
  *((int *)(dst + 2 * stride)) = _mm_cvtsi128_si32(p0);
  p0 = _mm_srli_si128(p0, 4);
  *((int *)(dst + 3 * stride)) = _mm_cvtsi128_si32(p0);
}

static void SavePred8x8(int *pred, const __m128i *mean, uint8_t *dst,
                        ptrdiff_t stride) {
  const int predStride = (maxBlkSize << 1) + 1;
  __m128i p0, p1, p2, p3;
  int r = 0;

  while (r < 8) {
    p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
    p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
    r += 1;
    p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
    p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));

    p0 = _mm_add_epi32(p0, mean[0]);
    p1 = _mm_add_epi32(p1, mean[0]);
    p2 = _mm_add_epi32(p2, mean[0]);
    p3 = _mm_add_epi32(p3, mean[0]);

    p0 = _mm_packus_epi32(p0, p1);
    p1 = _mm_packus_epi32(p2, p3);
    p0 = _mm_packus_epi16(p0, p1);

    _mm_storel_epi64((__m128i *)dst, p0);
    dst += stride;
    p0 = _mm_srli_si128(p0, 8);
    _mm_storel_epi64((__m128i *)dst, p0);
    dst += stride;
    r += 1;
  }
}

static void SavePred16x16(int *pred, const __m128i *mean, uint8_t *dst,
                          ptrdiff_t stride) {
  const int predStride = (maxBlkSize << 1) + 1;
  __m128i p0, p1, p2, p3;
  int r = 0;

  while (r < 16) {
    p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
    p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
    p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 8));
    p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 12));

    p0 = _mm_add_epi32(p0, mean[0]);
    p1 = _mm_add_epi32(p1, mean[0]);
    p2 = _mm_add_epi32(p2, mean[0]);
    p3 = _mm_add_epi32(p3, mean[0]);

    p0 = _mm_packus_epi32(p0, p1);
    p1 = _mm_packus_epi32(p2, p3);
    p0 = _mm_packus_epi16(p0, p1);

    _mm_storel_epi64((__m128i *)dst, p0);
    p0 = _mm_srli_si128(p0, 8);
    _mm_storel_epi64((__m128i *)(dst + 8), p0);
    dst += stride;
    r += 1;
  }
}

static void SavePred32x32(int *pred, const __m128i *mean, uint8_t *dst,
                          ptrdiff_t stride) {
  const int predStride = (maxBlkSize << 1) + 1;
  __m128i p0, p1, p2, p3, p4, p5, p6, p7;
  int r = 0;

  while (r < 32) {
    p0 = _mm_loadu_si128((const __m128i *)(pred + r * predStride));
    p1 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 4));
    p2 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 8));
    p3 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 12));

    p4 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 16));
    p5 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 20));
    p6 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 24));
    p7 = _mm_loadu_si128((const __m128i *)(pred + r * predStride + 28));

    p0 = _mm_add_epi32(p0, mean[0]);
    p1 = _mm_add_epi32(p1, mean[0]);
    p2 = _mm_add_epi32(p2, mean[0]);
    p3 = _mm_add_epi32(p3, mean[0]);

    p4 = _mm_add_epi32(p4, mean[0]);
    p5 = _mm_add_epi32(p5, mean[0]);
    p6 = _mm_add_epi32(p6, mean[0]);
    p7 = _mm_add_epi32(p7, mean[0]);

    p0 = _mm_packus_epi32(p0, p1);
    p1 = _mm_packus_epi32(p2, p3);
    p0 = _mm_packus_epi16(p0, p1);

    p4 = _mm_packus_epi32(p4, p5);
    p5 = _mm_packus_epi32(p6, p7);
    p4 = _mm_packus_epi16(p4, p5);

    _mm_storel_epi64((__m128i *)dst, p0);
    p0 = _mm_srli_si128(p0, 8);
    _mm_storel_epi64((__m128i *)(dst + 8), p0);

    _mm_storel_epi64((__m128i *)(dst + 16), p4);
    p4 = _mm_srli_si128(p4, 8);
    _mm_storel_epi64((__m128i *)(dst + 24), p4);

    dst += stride;
    r += 1;
  }
}

static void SavePrediction(int *pred, const __m128i *mean, int bs, uint8_t *dst,
                           ptrdiff_t stride) {
  switch (bs) {
    case 4: SavePred4x4(pred, mean, dst, stride); break;
    case 8: SavePred8x8(pred, mean, dst, stride); break;
    case 16: SavePred16x16(pred, mean, dst, stride); break;
    case 32: SavePred32x32(pred, mean, dst, stride); break;
    default: assert(0);
  }
}

typedef void (*ProducePixelsFunc)(__m128i *p, const __m128i *prm, int *pred,
                                  const int predStride);

static void ProduceFourPixels(__m128i *p, const __m128i *prm, int *pred,
                              const int predStride) {
  __m128i u0, u1, u2;
  int c0 = _mm_extract_epi32(prm[1], 0);
  int x = *(pred + predStride);
  int sum;

  u0 = _mm_mullo_epi32(p[0], prm[2]);
  u1 = _mm_mullo_epi32(p[1], prm[0]);
  u2 = _mm_mullo_epi32(p[2], prm[3]);

  u0 = _mm_add_epi32(u0, u1);
  u0 = _mm_add_epi32(u0, u2);

  sum = _mm_extract_epi32(u0, 0);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 1) = x;

  sum = _mm_extract_epi32(u0, 1);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 2) = x;

  sum = _mm_extract_epi32(u0, 2);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 3) = x;

  sum = _mm_extract_epi32(u0, 3);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 4) = x;
}

static void ProduceThreePixels(__m128i *p, const __m128i *prm, int *pred,
                               const int predStride) {
  __m128i u0, u1, u2;
  int c0 = _mm_extract_epi32(prm[1], 0);
  int x = *(pred + predStride);
  int sum;

  u0 = _mm_mullo_epi32(p[0], prm[2]);
  u1 = _mm_mullo_epi32(p[1], prm[0]);
  u2 = _mm_mullo_epi32(p[2], prm[3]);

  u0 = _mm_add_epi32(u0, u1);
  u0 = _mm_add_epi32(u0, u2);

  sum = _mm_extract_epi32(u0, 0);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 1) = x;

  sum = _mm_extract_epi32(u0, 1);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 2) = x;

  sum = _mm_extract_epi32(u0, 2);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 3) = x;
}

static void ProduceTwoPixels(__m128i *p, const __m128i *prm, int *pred,
                             const int predStride) {
  __m128i u0, u1, u2;
  int c0 = _mm_extract_epi32(prm[1], 0);
  int x = *(pred + predStride);
  int sum;

  u0 = _mm_mullo_epi32(p[0], prm[2]);
  u1 = _mm_mullo_epi32(p[1], prm[0]);
  u2 = _mm_mullo_epi32(p[2], prm[3]);

  u0 = _mm_add_epi32(u0, u1);
  u0 = _mm_add_epi32(u0, u2);

  sum = _mm_extract_epi32(u0, 0);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 1) = x;

  sum = _mm_extract_epi32(u0, 1);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 2) = x;
}

static void ProduceOnePixels(__m128i *p, const __m128i *prm, int *pred,
                             const int predStride) {
  __m128i u0, u1, u2;
  int c0 = _mm_extract_epi32(prm[1], 0);
  int x = *(pred + predStride);
  int sum;

  u0 = _mm_mullo_epi32(p[0], prm[2]);
  u1 = _mm_mullo_epi32(p[1], prm[0]);
  u2 = _mm_mullo_epi32(p[2], prm[3]);

  u0 = _mm_add_epi32(u0, u1);
  u0 = _mm_add_epi32(u0, u2);

  sum = _mm_extract_epi32(u0, 0);
  sum += c0 * x;
  x = ROUND_POWER_OF_TWO_SIGNED(sum, FILTER_INTRA_PREC_BITS);
  *(pred + predStride + 1) = x;
}

static ProducePixelsFunc prodPixelsFuncTab[4] = {
  ProduceOnePixels, ProduceTwoPixels, ProduceThreePixels, ProduceFourPixels
};

static void ProducePixels(int *pred, const __m128i *prm, int remain) {
  __m128i p[3];
  const int predStride = (maxBlkSize << 1) + 1;
  int index;

  p[0] = _mm_loadu_si128((const __m128i *)pred);
  p[1] = _mm_loadu_si128((const __m128i *)(pred + 1));
  p[2] = _mm_loadu_si128((const __m128i *)(pred + 2));

  if (remain <= 2) {
    return;
  }
  if (remain > 5) {
    index = 3;
  } else {
    index = remain - 3;
  }
  prodPixelsFuncTab[index](p, prm, pred, predStride);
}

// Note:
//  At column index c, the remaining pixels are R = 2 * bs + 1 - r - c
//  the number of pixels to produce is R - 2 = 2 * bs - r - c - 1
static void GeneratePrediction(const uint8_t *above, const uint8_t *left,
                               const int bs, const __m128i *prm, int meanValue,
                               uint8_t *dst, ptrdiff_t stride) {
  int pred[33][65];
  int r, c, colBound;
  int remainings;

  for (r = 0; r < bs; ++r) {
    pred[r + 1][0] = (int)left[r] - meanValue;
  }

  above -= 1;
  for (c = 0; c < 2 * bs + 1; ++c) {
    pred[0][c] = (int)above[c] - meanValue;
  }

  r = 0;
  c = 0;
  while (r < bs) {
    colBound = (bs << 1) - r;
    for (c = 0; c < colBound; c += 4) {
      remainings = colBound - c + 1;
      ProducePixels(&pred[r][c], prm, remainings);
    }
    r += 1;
  }

  SavePrediction(&pred[1][1], &prm[4], bs, dst, stride);
}

static void FilterPrediction(const uint8_t *above, const uint8_t *left, int bs,
                             __m128i *prm, uint8_t *dst, ptrdiff_t stride) {
  int meanValue = 0;
  meanValue = CalcRefPixelsMeanValue(above, left, bs, &prm[4]);
  GeneratePrediction(above, left, bs, prm, meanValue, dst, stride);
}

void av1_dc_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                    const uint8_t *above, const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, DC_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_v_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                   const uint8_t *above, const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, V_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_h_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                   const uint8_t *above, const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, H_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_d45_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                     const uint8_t *above,
                                     const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D45_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_d135_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                      const uint8_t *above,
                                      const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D135_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_d117_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                      const uint8_t *above,
                                      const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D117_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_d153_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                      const uint8_t *above,
                                      const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D153_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_d207_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                      const uint8_t *above,
                                      const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D207_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_d63_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                     const uint8_t *above,
                                     const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D63_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

void av1_tm_filter_predictor_sse4_1(uint8_t *dst, ptrdiff_t stride, int bs,
                                    const uint8_t *above, const uint8_t *left) {
  __m128i prm[5];
  GetIntraFilterParams(bs, TM_PRED, &prm[0]);
  FilterPrediction(above, left, bs, prm, dst, stride);
}

// ============== High Bit Depth ==============
#if CONFIG_AOM_HIGHBITDEPTH
static INLINE int HighbdGetMeanValue4x4(const uint16_t *above,
                                        const uint16_t *left, const int bd,
                                        __m128i *params) {
  const __m128i a = _mm_loadu_si128((const __m128i *)above);
  const __m128i l = _mm_loadu_si128((const __m128i *)left);
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector, u;
  uint16_t sum_value;
  (void)bd;

  sum_vector = _mm_add_epi16(a, l);

  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 2 values
  u = _mm_srli_si128(sum_vector, 2);
  sum_vector = _mm_add_epi16(sum_vector, u);

  sum_value = _mm_extract_epi16(sum_vector, 0);
  sum_value += 4;
  sum_value >>= 3;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

static INLINE int HighbdGetMeanValue8x8(const uint16_t *above,
                                        const uint16_t *left, const int bd,
                                        __m128i *params) {
  const __m128i a = _mm_loadu_si128((const __m128i *)above);
  const __m128i l = _mm_loadu_si128((const __m128i *)left);
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector, u;
  uint16_t sum_value;
  (void)bd;

  sum_vector = _mm_add_epi16(a, l);

  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 4 values
  sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 2 values

  u = _mm_srli_si128(sum_vector, 2);
  sum_vector = _mm_add_epi16(sum_vector, u);

  sum_value = _mm_extract_epi16(sum_vector, 0);
  sum_value += 8;
  sum_value >>= 4;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

// Note:
//  Process 16 pixels above and left, 10-bit depth
//  Add to the last 8 pixels sum
static INLINE void AddPixels10bit(const uint16_t *above, const uint16_t *left,
                                  __m128i *sum) {
  __m128i a = _mm_loadu_si128((const __m128i *)above);
  __m128i l = _mm_loadu_si128((const __m128i *)left);
  sum[0] = _mm_add_epi16(a, l);
  a = _mm_loadu_si128((const __m128i *)(above + 8));
  l = _mm_loadu_si128((const __m128i *)(left + 8));
  sum[0] = _mm_add_epi16(sum[0], a);
  sum[0] = _mm_add_epi16(sum[0], l);
}

// Note:
//  Process 16 pixels above and left, 12-bit depth
//  Add to the last 8 pixels sum
static INLINE void AddPixels12bit(const uint16_t *above, const uint16_t *left,
                                  __m128i *sum) {
  __m128i a = _mm_loadu_si128((const __m128i *)above);
  __m128i l = _mm_loadu_si128((const __m128i *)left);
  const __m128i zero = _mm_setzero_si128();
  __m128i v0, v1;

  v0 = _mm_unpacklo_epi16(a, zero);
  v1 = _mm_unpacklo_epi16(l, zero);
  sum[0] = _mm_add_epi32(v0, v1);

  v0 = _mm_unpackhi_epi16(a, zero);
  v1 = _mm_unpackhi_epi16(l, zero);
  sum[0] = _mm_add_epi32(sum[0], v0);
  sum[0] = _mm_add_epi32(sum[0], v1);

  a = _mm_loadu_si128((const __m128i *)(above + 8));
  l = _mm_loadu_si128((const __m128i *)(left + 8));

  v0 = _mm_unpacklo_epi16(a, zero);
  v1 = _mm_unpacklo_epi16(l, zero);
  sum[0] = _mm_add_epi32(sum[0], v0);
  sum[0] = _mm_add_epi32(sum[0], v1);

  v0 = _mm_unpackhi_epi16(a, zero);
  v1 = _mm_unpackhi_epi16(l, zero);
  sum[0] = _mm_add_epi32(sum[0], v0);
  sum[0] = _mm_add_epi32(sum[0], v1);
}

static INLINE int HighbdGetMeanValue16x16(const uint16_t *above,
                                          const uint16_t *left, const int bd,
                                          __m128i *params) {
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector, u;
  uint32_t sum_value = 0;

  if (10 == bd) {
    AddPixels10bit(above, left, &sum_vector);
    sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 4 values
    sum_vector = _mm_hadd_epi16(sum_vector, zero);  // still has 2 values

    u = _mm_srli_si128(sum_vector, 2);
    sum_vector = _mm_add_epi16(sum_vector, u);
    sum_value = _mm_extract_epi16(sum_vector, 0);
  } else if (12 == bd) {
    AddPixels12bit(above, left, &sum_vector);

    sum_vector = _mm_hadd_epi32(sum_vector, zero);
    u = _mm_srli_si128(sum_vector, 4);
    sum_vector = _mm_add_epi32(u, sum_vector);
    sum_value = _mm_extract_epi32(sum_vector, 0);
  }

  sum_value += 16;
  sum_value >>= 5;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

static INLINE int HighbdGetMeanValue32x32(const uint16_t *above,
                                          const uint16_t *left, const int bd,
                                          __m128i *params) {
  const __m128i zero = _mm_setzero_si128();
  __m128i sum_vector[2], u;
  uint32_t sum_value = 0;

  if (10 == bd) {
    AddPixels10bit(above, left, &sum_vector[0]);
    AddPixels10bit(above + 16, left + 16, &sum_vector[1]);

    sum_vector[0] = _mm_add_epi16(sum_vector[0], sum_vector[1]);
    sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero);  // still has 4 values
    sum_vector[0] = _mm_hadd_epi16(sum_vector[0], zero);  // still has 2 values

    u = _mm_srli_si128(sum_vector[0], 2);
    sum_vector[0] = _mm_add_epi16(sum_vector[0], u);
    sum_value = _mm_extract_epi16(sum_vector[0], 0);
  } else if (12 == bd) {
    AddPixels12bit(above, left, &sum_vector[0]);
    AddPixels12bit(above + 16, left + 16, &sum_vector[1]);

    sum_vector[0] = _mm_add_epi32(sum_vector[0], sum_vector[1]);
    sum_vector[0] = _mm_hadd_epi32(sum_vector[0], zero);
    u = _mm_srli_si128(sum_vector[0], 4);
    sum_vector[0] = _mm_add_epi32(u, sum_vector[0]);
    sum_value = _mm_extract_epi32(sum_vector[0], 0);
  }

  sum_value += 32;
  sum_value >>= 6;
  *params = _mm_set1_epi32(sum_value);
  return sum_value;
}

// Note:
//  params[4] : mean value, 4 int32_t repetition
//
static INLINE int HighbdCalcRefPixelsMeanValue(const uint16_t *above,
                                               const uint16_t *left, int bs,
                                               const int bd, __m128i *params) {
  int meanValue = 0;
  switch (bs) {
    case 4: meanValue = HighbdGetMeanValue4x4(above, left, bd, params); break;
    case 8: meanValue = HighbdGetMeanValue8x8(above, left, bd, params); break;
    case 16:
      meanValue = HighbdGetMeanValue16x16(above, left, bd, params);
      break;
    case 32:
      meanValue = HighbdGetMeanValue32x32(above, left, bd, params);
      break;
    default: assert(0);
  }
  return meanValue;
}

// Note:
//  At column index c, the remaining pixels are R = 2 * bs + 1 - r - c
//  the number of pixels to produce is R - 2 = 2 * bs - r - c - 1
static void HighbdGeneratePrediction(const uint16_t *above,
                                     const uint16_t *left, const int bs,
                                     const int bd, const __m128i *prm,
                                     int meanValue, uint16_t *dst,
                                     ptrdiff_t stride) {
  int pred[33][65];
  int r, c, colBound;
  int remainings;
  int ipred;

  for (r = 0; r < bs; ++r) {
    pred[r + 1][0] = (int)left[r] - meanValue;
  }

  above -= 1;
  for (c = 0; c < 2 * bs + 1; ++c) {
    pred[0][c] = (int)above[c] - meanValue;
  }

  r = 0;
  c = 0;
  while (r < bs) {
    colBound = (bs << 1) - r;
    for (c = 0; c < colBound; c += 4) {
      remainings = colBound - c + 1;
      ProducePixels(&pred[r][c], prm, remainings);
    }
    r += 1;
  }

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      ipred = pred[r + 1][c + 1] + meanValue;
      dst[c] = clip_pixel_highbd(ipred, bd);
    }
    dst += stride;
  }
}

static void HighbdFilterPrediction(const uint16_t *above, const uint16_t *left,
                                   int bs, const int bd, __m128i *prm,
                                   uint16_t *dst, ptrdiff_t stride) {
  int meanValue = 0;
  meanValue = HighbdCalcRefPixelsMeanValue(above, left, bs, bd, &prm[4]);
  HighbdGeneratePrediction(above, left, bs, bd, prm, meanValue, dst, stride);
}

void av1_highbd_dc_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                           int bs, const uint16_t *above,
                                           const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, DC_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_v_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                          int bs, const uint16_t *above,
                                          const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, V_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_h_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                          int bs, const uint16_t *above,
                                          const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, H_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_d45_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                            int bs, const uint16_t *above,
                                            const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D45_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_d135_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                             int bs, const uint16_t *above,
                                             const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D135_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_d117_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                             int bs, const uint16_t *above,
                                             const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D117_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_d153_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                             int bs, const uint16_t *above,
                                             const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D153_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_d207_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                             int bs, const uint16_t *above,
                                             const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D207_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_d63_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                            int bs, const uint16_t *above,
                                            const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, D63_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}

void av1_highbd_tm_filter_predictor_sse4_1(uint16_t *dst, ptrdiff_t stride,
                                           int bs, const uint16_t *above,
                                           const uint16_t *left, int bd) {
  __m128i prm[5];
  GetIntraFilterParams(bs, TM_PRED, &prm[0]);
  HighbdFilterPrediction(above, left, bs, bd, prm, dst, stride);
}
#endif  // CONFIG_AOM_HIGHBITDEPTH