blob: 0e805289f095ae947d70cc0008b65a815c31148e [file] [log] [blame]
/*
* Copyright (c) 2021, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 3-Clause Clear License
* and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
* License was not distributed with this source code in the LICENSE file, you
* can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the
* Alliance for Open Media Patent License 1.0 was not distributed with this
* source code in the PATENTS file, you can obtain it at
* aomedia.org/license/patent-license/.
*/
#include <arm_neon.h>
#include "common/tools_common.h"
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
//------------------------------------------------------------------------------
// DC 4x4
// 'do_above' and 'do_left' facilitate branch removal when inlined.
static INLINE void dc_4x4(uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
const uint8_t *left, int do_above, int do_left) {
uint16x8_t sum_top;
uint16x8_t sum_left;
uint8x8_t dc0;
if (do_above) {
const uint8x8_t A = vld1_u8(above); // top row
const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top
const uint16x4_t p1 = vpadd_u16(p0, p0);
sum_top = vcombine_u16(p1, p1);
}
if (do_left) {
const uint8x8_t L = vld1_u8(left); // left border
const uint16x4_t p0 = vpaddl_u8(L); // cascading summation of the left
const uint16x4_t p1 = vpadd_u16(p0, p0);
sum_left = vcombine_u16(p1, p1);
}
if (do_above && do_left) {
const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
dc0 = vrshrn_n_u16(sum, 3);
} else if (do_above) {
dc0 = vrshrn_n_u16(sum_top, 2);
} else if (do_left) {
dc0 = vrshrn_n_u16(sum_left, 2);
} else {
dc0 = vdup_n_u8(0x80);
}
{
const uint8x8_t dc = vdup_lane_u8(dc0, 0);
int i;
for (i = 0; i < 4; ++i) {
vst1_lane_u32((uint32_t *)(dst + i * stride), vreinterpret_u32_u8(dc), 0);
}
}
}
void aom_dc_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
dc_4x4(dst, stride, above, left, 1, 1);
}
void aom_dc_left_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)above;
dc_4x4(dst, stride, NULL, left, 0, 1);
}
void aom_dc_top_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)left;
dc_4x4(dst, stride, above, NULL, 1, 0);
}
void aom_dc_128_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)above;
(void)left;
dc_4x4(dst, stride, NULL, NULL, 0, 0);
}
//------------------------------------------------------------------------------
// DC 8x8
// 'do_above' and 'do_left' facilitate branch removal when inlined.
static INLINE void dc_8x8(uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
const uint8_t *left, int do_above, int do_left) {
uint16x8_t sum_top;
uint16x8_t sum_left;
uint8x8_t dc0;
if (do_above) {
const uint8x8_t A = vld1_u8(above); // top row
const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top
const uint16x4_t p1 = vpadd_u16(p0, p0);
const uint16x4_t p2 = vpadd_u16(p1, p1);
sum_top = vcombine_u16(p2, p2);
}
if (do_left) {
const uint8x8_t L = vld1_u8(left); // left border
const uint16x4_t p0 = vpaddl_u8(L); // cascading summation of the left
const uint16x4_t p1 = vpadd_u16(p0, p0);
const uint16x4_t p2 = vpadd_u16(p1, p1);
sum_left = vcombine_u16(p2, p2);
}
if (do_above && do_left) {
const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
dc0 = vrshrn_n_u16(sum, 4);
} else if (do_above) {
dc0 = vrshrn_n_u16(sum_top, 3);
} else if (do_left) {
dc0 = vrshrn_n_u16(sum_left, 3);
} else {
dc0 = vdup_n_u8(0x80);
}
{
const uint8x8_t dc = vdup_lane_u8(dc0, 0);
int i;
for (i = 0; i < 8; ++i) {
vst1_u32((uint32_t *)(dst + i * stride), vreinterpret_u32_u8(dc));
}
}
}
void aom_dc_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
dc_8x8(dst, stride, above, left, 1, 1);
}
void aom_dc_left_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)above;
dc_8x8(dst, stride, NULL, left, 0, 1);
}
void aom_dc_top_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)left;
dc_8x8(dst, stride, above, NULL, 1, 0);
}
void aom_dc_128_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
(void)above;
(void)left;
dc_8x8(dst, stride, NULL, NULL, 0, 0);
}
//------------------------------------------------------------------------------
// DC 16x16
// 'do_above' and 'do_left' facilitate branch removal when inlined.
static INLINE void dc_16x16(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left,
int do_above, int do_left) {
uint16x8_t sum_top;
uint16x8_t sum_left;
uint8x8_t dc0;
if (do_above) {
const uint8x16_t A = vld1q_u8(above); // top row
const uint16x8_t p0 = vpaddlq_u8(A); // cascading summation of the top
const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0));
const uint16x4_t p2 = vpadd_u16(p1, p1);
const uint16x4_t p3 = vpadd_u16(p2, p2);
sum_top = vcombine_u16(p3, p3);
}
if (do_left) {
const uint8x16_t L = vld1q_u8(left); // left row
const uint16x8_t p0 = vpaddlq_u8(L); // cascading summation of the left
const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0));
const uint16x4_t p2 = vpadd_u16(p1, p1);
const uint16x4_t p3 = vpadd_u16(p2, p2);
sum_left = vcombine_u16(p3, p3);
}
if (do_above && do_left) {
const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
dc0 = vrshrn_n_u16(sum, 5);
} else if (do_above) {
dc0 = vrshrn_n_u16(sum_top, 4);
} else if (do_left) {
dc0 = vrshrn_n_u16(sum_left, 4);
} else {
dc0 = vdup_n_u8(0x80);
}
{
const uint8x16_t dc = vdupq_lane_u8(dc0, 0);
int i;
for (i = 0; i < 16; ++i) {
vst1q_u8(dst + i * stride, dc);
}
}
}
void aom_dc_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
dc_16x16(dst, stride, above, left, 1, 1);
}
void aom_dc_left_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
(void)above;
dc_16x16(dst, stride, NULL, left, 0, 1);
}
void aom_dc_top_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
(void)left;
dc_16x16(dst, stride, above, NULL, 1, 0);
}
void aom_dc_128_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
(void)above;
(void)left;
dc_16x16(dst, stride, NULL, NULL, 0, 0);
}
//------------------------------------------------------------------------------
// DC 32x32
// 'do_above' and 'do_left' facilitate branch removal when inlined.
static INLINE void dc_32x32(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left,
int do_above, int do_left) {
uint16x8_t sum_top;
uint16x8_t sum_left;
uint8x8_t dc0;
if (do_above) {
const uint8x16_t A0 = vld1q_u8(above); // top row
const uint8x16_t A1 = vld1q_u8(above + 16);
const uint16x8_t p0 = vpaddlq_u8(A0); // cascading summation of the top
const uint16x8_t p1 = vpaddlq_u8(A1);
const uint16x8_t p2 = vaddq_u16(p0, p1);
const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2));
const uint16x4_t p4 = vpadd_u16(p3, p3);
const uint16x4_t p5 = vpadd_u16(p4, p4);
sum_top = vcombine_u16(p5, p5);
}
if (do_left) {
const uint8x16_t L0 = vld1q_u8(left); // left row
const uint8x16_t L1 = vld1q_u8(left + 16);
const uint16x8_t p0 = vpaddlq_u8(L0); // cascading summation of the left
const uint16x8_t p1 = vpaddlq_u8(L1);
const uint16x8_t p2 = vaddq_u16(p0, p1);
const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2));
const uint16x4_t p4 = vpadd_u16(p3, p3);
const uint16x4_t p5 = vpadd_u16(p4, p4);
sum_left = vcombine_u16(p5, p5);
}
if (do_above && do_left) {
const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
dc0 = vrshrn_n_u16(sum, 6);
} else if (do_above) {
dc0 = vrshrn_n_u16(sum_top, 5);
} else if (do_left) {
dc0 = vrshrn_n_u16(sum_left, 5);
} else {
dc0 = vdup_n_u8(0x80);
}
{
const uint8x16_t dc = vdupq_lane_u8(dc0, 0);
int i;
for (i = 0; i < 32; ++i) {
vst1q_u8(dst + i * stride, dc);
vst1q_u8(dst + i * stride + 16, dc);
}
}
}
void aom_dc_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
dc_32x32(dst, stride, above, left, 1, 1);
}
void aom_dc_left_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
(void)above;
dc_32x32(dst, stride, NULL, left, 0, 1);
}
void aom_dc_top_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
(void)left;
dc_32x32(dst, stride, above, NULL, 1, 0);
}
void aom_dc_128_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
(void)above;
(void)left;
dc_32x32(dst, stride, NULL, NULL, 0, 0);
}
// -----------------------------------------------------------------------------
void aom_d135_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
const uint8x8_t XABCD_u8 = vld1_u8(above - 1);
const uint64x1_t XABCD = vreinterpret_u64_u8(XABCD_u8);
const uint64x1_t ____XABC = vshl_n_u64(XABCD, 32);
const uint32x2_t zero = vdup_n_u32(0);
const uint32x2_t IJKL = vld1_lane_u32((const uint32_t *)left, zero, 0);
const uint8x8_t IJKL_u8 = vreinterpret_u8_u32(IJKL);
const uint64x1_t LKJI____ = vreinterpret_u64_u8(vrev32_u8(IJKL_u8));
const uint64x1_t LKJIXABC = vorr_u64(LKJI____, ____XABC);
const uint8x8_t KJIXABC_ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 8));
const uint8x8_t JIXABC__ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 16));
const uint8_t D = vget_lane_u8(XABCD_u8, 4);
const uint8x8_t JIXABCD_ = vset_lane_u8(D, JIXABC__, 6);
const uint8x8_t LKJIXABC_u8 = vreinterpret_u8_u64(LKJIXABC);
const uint8x8_t avg1 = vhadd_u8(JIXABCD_, LKJIXABC_u8);
const uint8x8_t avg2 = vrhadd_u8(avg1, KJIXABC_);
const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2);
const uint32x2_t r3 = vreinterpret_u32_u8(avg2);
const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8));
const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16));
const uint32x2_t r0 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24));
vst1_lane_u32((uint32_t *)(dst + 0 * stride), r0, 0);
vst1_lane_u32((uint32_t *)(dst + 1 * stride), r1, 0);
vst1_lane_u32((uint32_t *)(dst + 2 * stride), r2, 0);
vst1_lane_u32((uint32_t *)(dst + 3 * stride), r3, 0);
}
void aom_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int i;
uint32x2_t d0u32 = vdup_n_u32(0);
(void)left;
d0u32 = vld1_lane_u32((const uint32_t *)above, d0u32, 0);
for (i = 0; i < 4; i++, dst += stride)
vst1_lane_u32((uint32_t *)dst, d0u32, 0);
}
void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int i;
uint8x8_t d0u8 = vdup_n_u8(0);
(void)left;
d0u8 = vld1_u8(above);
for (i = 0; i < 8; i++, dst += stride) vst1_u8(dst, d0u8);
}
void aom_v_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int i;
uint8x16_t q0u8 = vdupq_n_u8(0);
(void)left;
q0u8 = vld1q_u8(above);
for (i = 0; i < 16; i++, dst += stride) vst1q_u8(dst, q0u8);
}
void aom_v_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int i;
uint8x16_t q0u8 = vdupq_n_u8(0);
uint8x16_t q1u8 = vdupq_n_u8(0);
(void)left;
q0u8 = vld1q_u8(above);
q1u8 = vld1q_u8(above + 16);
for (i = 0; i < 32; i++, dst += stride) {
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q1u8);
}
}
void aom_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x8_t d0u8 = vdup_n_u8(0);
uint32x2_t d1u32 = vdup_n_u32(0);
(void)above;
d1u32 = vld1_lane_u32((const uint32_t *)left, d1u32, 0);
d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 0);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 1);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 2);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 3);
vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
}
void aom_h_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x8_t d0u8 = vdup_n_u8(0);
uint64x1_t d1u64 = vdup_n_u64(0);
(void)above;
d1u64 = vld1_u64((const uint64_t *)left);
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 0);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 1);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 2);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 3);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 4);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 5);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 6);
vst1_u8(dst, d0u8);
dst += stride;
d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 7);
vst1_u8(dst, d0u8);
}
void aom_h_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int j;
uint8x8_t d2u8 = vdup_n_u8(0);
uint8x16_t q0u8 = vdupq_n_u8(0);
uint8x16_t q1u8 = vdupq_n_u8(0);
(void)above;
q1u8 = vld1q_u8(left);
d2u8 = vget_low_u8(q1u8);
for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) {
q0u8 = vdupq_lane_u8(d2u8, 0);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 1);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 2);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 3);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 4);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 5);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 6);
vst1q_u8(dst, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 7);
vst1q_u8(dst, q0u8);
dst += stride;
}
}
void aom_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
int j, k;
uint8x8_t d2u8 = vdup_n_u8(0);
uint8x16_t q0u8 = vdupq_n_u8(0);
uint8x16_t q1u8 = vdupq_n_u8(0);
(void)above;
for (k = 0; k < 2; k++, left += 16) {
q1u8 = vld1q_u8(left);
d2u8 = vget_low_u8(q1u8);
for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) {
q0u8 = vdupq_lane_u8(d2u8, 0);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 1);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 2);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 3);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 4);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 5);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 6);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
q0u8 = vdupq_lane_u8(d2u8, 7);
vst1q_u8(dst, q0u8);
vst1q_u8(dst + 16, q0u8);
dst += stride;
}
}
}
static INLINE void highbd_dc_predictor(uint16_t *dst, ptrdiff_t stride, int bw,
const uint16_t *above,
const uint16_t *left) {
assert(bw >= 4);
assert(IS_POWER_OF_TWO(bw));
int expected_dc, sum = 0;
const int count = bw * 2;
uint32x4_t sum_q = vdupq_n_u32(0);
uint32x2_t sum_d;
uint16_t *dst_1;
if (bw >= 8) {
for (int i = 0; i < bw; i += 8) {
sum_q = vpadalq_u16(sum_q, vld1q_u16(above));
sum_q = vpadalq_u16(sum_q, vld1q_u16(left));
above += 8;
left += 8;
}
sum_d = vadd_u32(vget_low_u32(sum_q), vget_high_u32(sum_q));
sum = vget_lane_s32(vreinterpret_s32_u64(vpaddl_u32(sum_d)), 0);
expected_dc = (sum + (count >> 1)) / count;
const uint16x8_t dc = vdupq_n_u16((uint16_t)expected_dc);
for (int r = 0; r < bw; r++) {
dst_1 = dst;
for (int i = 0; i < bw; i += 8) {
vst1q_u16(dst_1, dc);
dst_1 += 8;
}
dst += stride;
}
} else { // 4x4
sum_q = vaddl_u16(vld1_u16(above), vld1_u16(left));
sum_d = vadd_u32(vget_low_u32(sum_q), vget_high_u32(sum_q));
sum = vget_lane_s32(vreinterpret_s32_u64(vpaddl_u32(sum_d)), 0);
expected_dc = (sum + (count >> 1)) / count;
const uint16x4_t dc = vdup_n_u16((uint16_t)expected_dc);
for (int r = 0; r < bw; r++) {
vst1_u16(dst, dc);
dst += stride;
}
}
}
#define intra_pred_highbd_sized_neon(type, width) \
void aom_highbd_##type##_predictor_##width##x##width##_neon( \
uint16_t *dst, ptrdiff_t stride, const uint16_t *above, \
const uint16_t *left, int bd) { \
(void)bd; \
highbd_##type##_predictor(dst, stride, width, above, left); \
}
#define intra_pred_square(type) \
intra_pred_highbd_sized_neon(type, 4); \
intra_pred_highbd_sized_neon(type, 8); \
intra_pred_highbd_sized_neon(type, 16); \
intra_pred_highbd_sized_neon(type, 32); \
intra_pred_highbd_sized_neon(type, 64);
intra_pred_square(dc);
#undef intra_pred_square
static const int sm_weight_log2_scale = 8;
// max(block_size_wide[BLOCK_LARGEST], block_size_high[BLOCK_LARGEST])
#define MAX_BLOCK_DIM 64
/* clang-format off */
static const uint8_t sm_weight_arrays[2 * MAX_BLOCK_DIM] = {
// Unused, because we always offset by bs, which is at least 2.
0, 0,
// bs = 2
255, 128,
// bs = 4
255, 149, 85, 64,
// bs = 8
255, 197, 146, 105, 73, 50, 37, 32,
// bs = 16
255, 225, 196, 170, 145, 123, 102, 84, 68, 54, 43, 33, 26, 20, 17, 16,
// bs = 32
255, 240, 225, 210, 196, 182, 169, 157, 145, 133, 122, 111, 101, 92, 83, 74,
66, 59, 52, 45, 39, 34, 29, 25, 21, 17, 14, 12, 10, 9, 8, 8,
// bs = 64
255, 248, 240, 233, 225, 218, 210, 203, 196, 189, 182, 176, 169, 163, 156,
150, 144, 138, 133, 127, 121, 116, 111, 106, 101, 96, 91, 86, 82, 77, 73,
69, 65, 61, 57, 54, 50, 47, 44, 41, 38, 35, 32, 29, 27, 25, 22, 20, 18, 16,
15, 13, 12, 10, 9, 8, 7, 6, 6, 5, 5, 4, 4, 4,
};
/* clang-format on */
// -----------------------------------------------------------------------------
// SMOOTH_PRED
// pixels[0]: above and below_pred interleave vector
// pixels[1]: left vector
// pixels[2]: right_pred vector
static INLINE void load_pixel_w4(const uint8_t *above, const uint8_t *left,
int height, uint8x16_t *pixels) {
uint32x4_t zero = vdupq_n_u32(0);
const uint8x8_t d = vcreate_u8(((const uint32_t *)above)[0]);
if (height == 4)
pixels[1] =
vreinterpretq_u8_u32(vld1q_lane_u32((const uint32_t *)left, zero, 0));
else if (height == 8) {
pixels[1] = vreinterpretq_u8_u64(vsetq_lane_u64(
((const uint64_t *)left)[0], vreinterpretq_u64_u32(zero), 0));
} else {
pixels[1] = vld1q_u8(left);
}
pixels[2] = vreinterpretq_u8_u16(vdupq_n_u16(above[3]));
const uint16x8_t bp = vdupq_n_u16(left[height - 1]);
#if defined(__aarch64__)
pixels[0] = vreinterpretq_u8_u16(vzip1q_u16(vmovl_u8(d), bp));
#else
pixels[0] = vreinterpretq_u8_u16(vzipq_u16(vmovl_u8(d), bp).val[0]);
#endif // (__aarch64__)
}
// weight_h[0]: weight_h vector
// weight_h[1]: scale - weight_h vector
// weight_h[2]: same as [0], second half for height = 16 only
// weight_h[3]: same as [1], second half for height = 16 only
// weight_w[0]: weights_w and scale - weights_w interleave vector
static INLINE void load_weight_w4(const uint8_t *weight_array, int height,
uint16x8_t *weight_h, uint16x8_t *weight_w) {
const uint16x8_t d = vdupq_n_u16((uint16_t)(1 << sm_weight_log2_scale));
const uint8x8_t t = vcreate_u8(((const uint32_t *)(weight_array))[1]);
weight_h[0] = vmovl_u8(t);
weight_h[1] = vsubw_u8(d, t);
#if defined(__aarch64__)
weight_w[0] = vzip1q_u16(weight_h[0], weight_h[1]);
#else
weight_w[0] = vzipq_u16(weight_h[0], weight_h[1]).val[0];
#endif // (__aarch64__)
if (height == 8) {
const uint8x8_t weight = vld1_u8(&weight_array[8]);
weight_h[0] = vmovl_u8(weight);
weight_h[1] = vsubw_u8(d, weight);
} else if (height == 16) {
const uint8x16_t zero = vdupq_n_u8(0);
const uint8x16_t weight = vld1q_u8(&weight_array[16]);
const uint8x16x2_t weight_h_02 = vzipq_u8(weight, zero);
weight_h[0] = vreinterpretq_u16_u8(weight_h_02.val[0]);
weight_h[1] = vsubq_u16(d, vreinterpretq_u16_u8(weight_h_02.val[0]));
weight_h[2] = vreinterpretq_u16_u8(weight_h_02.val[1]);
weight_h[3] = vsubq_u16(d, vreinterpretq_u16_u8(weight_h_02.val[1]));
}
}
static INLINE void smooth_pred_4xh(const uint8x16_t *pixel,
const uint16x8_t *wh, const uint16x8_t *ww,
int h, uint8_t *dst, ptrdiff_t stride,
int second_half) {
const uint16x4_t one = vdup_n_u16(1);
const uint16x4_t inc = vdup_n_u16(0x202);
uint16x4_t rep =
second_half ? vdup_n_u16((uint16_t)0x8008) : vdup_n_u16((uint16_t)0x8000);
uint16x4_t d = vdup_n_u16(0x100);
const uint16x4_t v_pixel_0_lo = vmovn_u32(vreinterpretq_u32_u8(pixel[0]));
const uint16x4_t v_pixel_0_hi =
vmovn_u32(vreinterpretq_u32_u8(vextq_u8(pixel[0], pixel[0], 2)));
const uint16x4_t v_pixel_2 = vget_low_u16(vreinterpretq_u16_u8(pixel[2]));
const uint16x4_t ww_0_lo = vmovn_u32(vreinterpretq_u32_u16(ww[0]));
const uint16x4_t ww_0_hi =
vmovn_u32(vreinterpretq_u32_u16(vextq_u16(ww[0], ww[0], 1)));
const uint8x8_t save_mask = vcreate_u8(0 + (2 << 8) + (4 << 16) + (6 << 24));
#if !defined(__aarch64__)
const uint8x8x2_t v_split1 = { { vget_low_u8(vreinterpretq_u8_u16(wh[0])),
vget_high_u8(
vreinterpretq_u8_u16(wh[0])) } };
const uint8x8x2_t v_split2 = { { vget_low_u8(vreinterpretq_u8_u16(wh[1])),
vget_high_u8(
vreinterpretq_u8_u16(wh[1])) } };
const uint8x8x2_t v_split3 = { { vget_low_u8(pixel[1]),
vget_high_u8(pixel[1]) } };
#endif // (__aarch64__)
for (int i = 0; i < h; ++i) {
#if defined(__aarch64__)
const uint8x8_t wg =
vqtbl1_u8(vreinterpretq_u8_u16(wh[0]), vreinterpret_u8_u16(d));
const uint8x8_t sc =
vqtbl1_u8(vreinterpretq_u8_u16(wh[1]), vreinterpret_u8_u16(d));
#else
const uint8x8_t wg = vtbl2_u8(v_split1, vreinterpret_u8_u16(d));
const uint8x8_t sc = vtbl2_u8(v_split2, vreinterpret_u8_u16(d));
#endif // (__aarch64__)
uint32x4_t sum = vmull_u16(v_pixel_0_lo, vreinterpret_u16_u8(wg));
sum = vmlal_u16(sum, v_pixel_0_hi, vreinterpret_u16_u8(sc));
#if defined(__aarch64__)
uint8x8_t b = vqtbl1_u8(pixel[1], vreinterpret_u8_u16(rep));
#else
uint8x8_t b = vtbl2_u8(v_split3, vreinterpret_u8_u16(rep));
#endif // (__aarch64__)
sum = vmlal_u16(sum, vreinterpret_u16_u8(b), ww_0_lo);
sum = vmlal_u16(sum, v_pixel_2, ww_0_hi);
uint8x8_t sum_l = vreinterpret_u8_u16(vqrshrn_n_u32(sum, 9));
uint32x2_t predsh = vreinterpret_u32_u8(vtbl1_u8(sum_l, save_mask));
vst1_lane_u32((uint32_t *)dst, predsh, 0);
dst += stride;
rep = vadd_u16(rep, one);
d = vadd_u16(d, inc);
}
}
void aom_smooth_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[3];
load_pixel_w4(above, left, 4, pixels);
uint16x8_t wh[4], ww[2];
load_weight_w4(sm_weight_arrays, 4, wh, ww);
smooth_pred_4xh(pixels, wh, ww, 4, dst, stride, 0);
}
void aom_smooth_predictor_4x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[3];
load_pixel_w4(above, left, 8, pixels);
uint16x8_t wh[4], ww[2];
load_weight_w4(sm_weight_arrays, 8, wh, ww);
smooth_pred_4xh(pixels, wh, ww, 8, dst, stride, 0);
}
void aom_smooth_predictor_4x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[3];
load_pixel_w4(above, left, 16, pixels);
uint16x8_t wh[4], ww[2];
load_weight_w4(sm_weight_arrays, 16, wh, ww);
smooth_pred_4xh(pixels, wh, ww, 8, dst, stride, 0);
dst += stride << 3;
smooth_pred_4xh(pixels, &wh[2], ww, 8, dst, stride, 1);
}
// pixels[0]: above and below_pred interleave vector, first half
// pixels[1]: above and below_pred interleave vector, second half
// pixels[2]: left vector
// pixels[3]: right_pred vector
// pixels[4]: above and below_pred interleave vector, first half
// pixels[5]: above and below_pred interleave vector, second half
// pixels[6]: left vector + 16
// pixels[7]: right_pred vector
static INLINE void load_pixel_w8(const uint8_t *above, const uint8_t *left,
int height, uint8x16_t *pixels) {
pixels[0] = vreinterpretq_u8_u16(vmovl_u8(vld1_u8(above)));
pixels[1] = vreinterpretq_u8_u16(vdupq_n_u16((uint16_t)left[height - 1]));
pixels[3] = vreinterpretq_u8_u16(vdupq_n_u16((uint16_t)above[7]));
if (height == 4) {
const uint32x4_t zero32 = vdupq_n_u32(0);
pixels[2] =
vreinterpretq_u8_u32(vld1q_lane_u32((const uint32_t *)left, zero32, 0));
} else if (height == 8) {
const uint64x2_t zero64 = vdupq_n_u64(0);
pixels[2] = vreinterpretq_u8_u64(
vsetq_lane_u64(((const uint64_t *)left)[0], zero64, 0));
} else if (height == 16) {
pixels[2] = vld1q_u8(left);
} else {
pixels[2] = vld1q_u8(left);
pixels[4] = pixels[0];
pixels[5] = pixels[1];
pixels[6] = vld1q_u8(left + 16);
pixels[7] = pixels[3];
}
}
// weight_h[0]: weight_h vector
// weight_h[1]: scale - weight_h vector
// weight_h[2]: same as [0], offset 8
// weight_h[3]: same as [1], offset 8
// weight_h[4]: same as [0], offset 16
// weight_h[5]: same as [1], offset 16
// weight_h[6]: same as [0], offset 24
// weight_h[7]: same as [1], offset 24
// weight_w[0]: weights_w and scale - weights_w interleave vector, first half
// weight_w[1]: weights_w and scale - weights_w interleave vector, second half
static INLINE void load_weight_w8(const uint8_t *weight_array, int height,
uint16x8_t *weight_h, uint16x8_t *weight_w) {
const uint8x16_t zero = vdupq_n_u8(0);
const int we_offset = height < 8 ? 4 : 8;
uint8x16_t we = vld1q_u8(&weight_array[we_offset]);
#if defined(__aarch64__)
weight_h[0] = vreinterpretq_u16_u8(vzip1q_u8(we, zero));
#else
weight_h[0] = vreinterpretq_u16_u8(vzipq_u8(we, zero).val[0]);
#endif // (__aarch64__)
const uint16x8_t d = vdupq_n_u16(256);
weight_h[1] = vsubq_u16(d, weight_h[0]);
if (height == 4) {
we = vextq_u8(we, zero, 4);
#if defined(__aarch64__)
weight_w[0] = vreinterpretq_u16_u8(vzip1q_u8(we, zero));
#else
weight_w[0] = vmovl_u8(vget_low_u8(we));
#endif // (__aarch64__)
weight_w[1] = vsubq_u16(d, weight_w[0]);
} else {
weight_w[0] = weight_h[0];
weight_w[1] = weight_h[1];
}
if (height == 16) {
we = vld1q_u8(&weight_array[16]);
const uint8x16x2_t weight_h_02 = vzipq_u8(we, zero);
weight_h[0] = vreinterpretq_u16_u8(weight_h_02.val[0]);
weight_h[1] = vsubq_u16(d, weight_h[0]);
weight_h[2] = vreinterpretq_u16_u8(weight_h_02.val[1]);
weight_h[3] = vsubq_u16(d, weight_h[2]);
} else if (height == 32) {
const uint8x16_t weight_lo = vld1q_u8(&weight_array[32]);
const uint8x16x2_t weight_h_02 = vzipq_u8(weight_lo, zero);
weight_h[0] = vreinterpretq_u16_u8(weight_h_02.val[0]);
weight_h[1] = vsubq_u16(d, weight_h[0]);
weight_h[2] = vreinterpretq_u16_u8(weight_h_02.val[1]);
weight_h[3] = vsubq_u16(d, weight_h[2]);
const uint8x16_t weight_hi = vld1q_u8(&weight_array[32 + 16]);
const uint8x16x2_t weight_h_46 = vzipq_u8(weight_hi, zero);
weight_h[4] = vreinterpretq_u16_u8(weight_h_46.val[0]);
weight_h[5] = vsubq_u16(d, weight_h[4]);
weight_h[6] = vreinterpretq_u16_u8(weight_h_46.val[1]);
weight_h[7] = vsubq_u16(d, weight_h[6]);
}
}
static INLINE void smooth_pred_8xh(const uint8x16_t *pixels,
const uint16x8_t *wh, const uint16x8_t *ww,
int h, uint8_t *dst, ptrdiff_t stride,
int second_half) {
const uint16x8_t one = vdupq_n_u16(1);
const uint16x8_t inc = vdupq_n_u16(0x202);
uint16x8_t rep = second_half ? vdupq_n_u16((uint16_t)0x8008)
: vdupq_n_u16((uint16_t)0x8000);
uint16x8_t d = vdupq_n_u16(0x100);
#if !defined(__aarch64__)
const uint8x8x2_t v_split1 = { { vget_low_u8(vreinterpretq_u8_u16(wh[0])),
vget_high_u8(
vreinterpretq_u8_u16(wh[0])) } };
const uint8x8x2_t v_split2 = { { vget_low_u8(vreinterpretq_u8_u16(wh[1])),
vget_high_u8(
vreinterpretq_u8_u16(wh[1])) } };
const uint8x8x2_t v_split3 = { { vget_low_u8(pixels[2]),
vget_high_u8(pixels[2]) } };
#endif
for (int i = 0; i < h; ++i) {
#if defined(__aarch64__)
const uint8x16_t wg_wg =
vqtbl1q_u8(vreinterpretq_u8_u16(wh[0]), vreinterpretq_u8_u16(d));
const uint8x16_t sc_sc =
vqtbl1q_u8(vreinterpretq_u8_u16(wh[1]), vreinterpretq_u8_u16(d));
#else
const uint8x8_t v_d_lo = vreinterpret_u8_u16(vget_low_u16(d));
const uint8x8_t v_d_hi = vreinterpret_u8_u16(vget_high_u16(d));
const uint8x16_t wg_wg =
vcombine_u8(vtbl2_u8(v_split1, v_d_lo), vtbl2_u8(v_split1, v_d_hi));
const uint8x16_t sc_sc =
vcombine_u8(vtbl2_u8(v_split2, v_d_lo), vtbl2_u8(v_split2, v_d_hi));
#endif // (__aarch64__)
uint16x8_t s01 =
vmulq_u16(vreinterpretq_u16_u8(pixels[0]), vreinterpretq_u16_u8(wg_wg));
s01 = vmlaq_u16(s01, vreinterpretq_u16_u8(pixels[1]),
vreinterpretq_u16_u8(sc_sc));
#if defined(__aarch64__)
const uint8x16_t b = vqtbl1q_u8(pixels[2], vreinterpretq_u8_u16(rep));
#else
const uint8x16_t b = vcombine_u8(
vtbl2_u8(v_split3, vget_low_u8(vreinterpretq_u8_u16(rep))),
vtbl2_u8(v_split3, vget_high_u8(vreinterpretq_u8_u16(rep))));
#endif // (__aarch64__)
uint16x8_t sum0 = vmulq_u16(vreinterpretq_u16_u8(b), ww[0]);
sum0 = vmlaq_u16(sum0, vreinterpretq_u16_u8(pixels[3]), ww[1]);
uint32x4_t s0 = vaddl_u16(vget_low_u16(s01), vget_low_u16(sum0));
#if defined(__aarch64__)
uint32x4_t s1 = vaddl_high_u16(s01, sum0);
#else
uint32x4_t s1 = vaddl_u16(vget_high_u16(s01), vget_high_u16(sum0));
#endif // (__aarch64__)
sum0 = vcombine_u16(vqrshrn_n_u32(s0, 9), vqrshrn_n_u32(s1, 9));
uint8x8_t predsh = vqmovn_u16(sum0);
vst1_u8(dst, predsh);
dst += stride;
rep = vaddq_u16(rep, one);
d = vaddq_u16(d, inc);
}
}
void aom_smooth_predictor_8x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[4];
load_pixel_w8(above, left, 4, pixels);
uint16x8_t wh[4], ww[2];
load_weight_w8(sm_weight_arrays, 4, wh, ww);
smooth_pred_8xh(pixels, wh, ww, 4, dst, stride, 0);
}
void aom_smooth_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[4];
load_pixel_w8(above, left, 8, pixels);
uint16x8_t wh[4], ww[2];
load_weight_w8(sm_weight_arrays, 8, wh, ww);
smooth_pred_8xh(pixels, wh, ww, 8, dst, stride, 0);
}
void aom_smooth_predictor_8x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[4];
load_pixel_w8(above, left, 16, pixels);
uint16x8_t wh[4], ww[2];
load_weight_w8(sm_weight_arrays, 16, wh, ww);
smooth_pred_8xh(pixels, wh, ww, 8, dst, stride, 0);
dst += stride << 3;
smooth_pred_8xh(pixels, &wh[2], ww, 8, dst, stride, 1);
}
void aom_smooth_predictor_8x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
uint8x16_t pixels[8];
load_pixel_w8(above, left, 32, pixels);
uint16x8_t wh[8], ww[2];
load_weight_w8(sm_weight_arrays, 32, wh, ww);
smooth_pred_8xh(&pixels[0], wh, ww, 8, dst, stride, 0);
dst += stride << 3;
smooth_pred_8xh(&pixels[0], &wh[2], ww, 8, dst, stride, 1);
dst += stride << 3;
smooth_pred_8xh(&pixels[4], &wh[4], ww, 8, dst, stride, 0);
dst += stride << 3;
smooth_pred_8xh(&pixels[4], &wh[6], ww, 8, dst, stride, 1);
}
static INLINE void smooth_predictor_wxh(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left, uint32_t bw,
uint32_t bh) {
const uint8_t *const sm_weights_w = sm_weight_arrays + bw;
const uint8_t *const sm_weights_h = sm_weight_arrays + bh;
const uint16x8_t scale_value = vdupq_n_u16(256);
for (uint32_t y = 0; y < bh; ++y) {
const uint8x8_t left_y = vdup_n_u8(left[y]);
const uint8x8_t weights_y_dup = vdup_n_u8(sm_weights_h[y]);
const uint32x4_t pred_scaled_bl =
vdupq_n_u32(256 + (256 - sm_weights_h[y]) * left[bh - 1]);
for (uint32_t x = 0; x < bw; x += 8) {
const uint8x8_t weights_x = vld1_u8(sm_weights_w + x);
const uint8x8_t top_x = vld1_u8(above + x);
uint16x8_t pred_m1, pred_m2;
uint32x4_t pred_lo, pred_hi;
pred_m1 = vmull_u8(top_x, weights_y_dup);
pred_m2 = vmull_u8(weights_x, left_y);
pred_lo = vaddl_u16(vget_low_u16(pred_m1), vget_low_u16(pred_m2));
#if defined(__aarch64__)
pred_hi = vaddl_high_u16(pred_m1, pred_m2);
#else
pred_hi = vaddl_u16(vget_high_u16(pred_m1), vget_high_u16(pred_m2));
#endif // (__aarch64__)
const uint16x8_t scale_m_weights_x = vsubw_u8(scale_value, weights_x);
const uint16x8_t swxtr = vmulq_n_u16(scale_m_weights_x, above[bw - 1]);
pred_lo = vaddq_u32(pred_lo, pred_scaled_bl);
pred_hi = vaddq_u32(pred_hi, pred_scaled_bl);
pred_lo = vaddw_u16(pred_lo, vget_low_u16(swxtr));
#if defined(__aarch64__)
pred_hi = vaddw_high_u16(pred_hi, swxtr);
#else
pred_hi = vaddw_u16(pred_hi, vget_high_u16(swxtr));
#endif // (__aarch64__)
uint16x8_t pred =
vcombine_u16(vshrn_n_u32(pred_lo, 9), vshrn_n_u32(pred_hi, 9));
uint8x8_t predsh = vqmovn_u16(pred);
vst1_u8(dst + x, predsh);
}
dst += stride;
}
}
void aom_smooth_predictor_16x4_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 16, 4);
}
void aom_smooth_predictor_16x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 16, 8);
}
void aom_smooth_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 16, 16);
}
void aom_smooth_predictor_16x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 16, 32);
}
void aom_smooth_predictor_32x8_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above, const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 32, 8);
}
void aom_smooth_predictor_32x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 32, 16);
}
void aom_smooth_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 32, 32);
}
void aom_smooth_predictor_32x64_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 32, 64);
}
void aom_smooth_predictor_64x64_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 64, 64);
}
void aom_smooth_predictor_64x32_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 64, 32);
}
void aom_smooth_predictor_64x16_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 64, 16);
}
void aom_smooth_predictor_16x64_neon(uint8_t *dst, ptrdiff_t stride,
const uint8_t *above,
const uint8_t *left) {
smooth_predictor_wxh(dst, stride, above, left, 16, 64);
}