Add ARM Neon optimization of txfm 8x8 2 % gain in decoder time. Change-Id: I17c3c92533b9809b308c7dc621d607570346c0a2
diff --git a/av1/common/arm/av1_inv_txfm_neon.c b/av1/common/arm/av1_inv_txfm_neon.c index 51c9914..2692f03 100644 --- a/av1/common/arm/av1_inv_txfm_neon.c +++ b/av1/common/arm/av1_inv_txfm_neon.c
@@ -9,6 +9,8 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#include <arm_neon.h> + #include "config/aom_config.h" #include "config/aom_dsp_rtcd.h" #include "config/av1_rtcd.h" @@ -19,6 +21,7 @@ #include "av1/common/enums.h" #include "av1/common/idct.h" #include "av1/common/arm/av1_inv_txfm_neon.h" +#include "av1/common/arm/transpose_neon.h" static INLINE TxSetType find_TxSetType(TX_SIZE tx_size) { const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size]; @@ -65,6 +68,443 @@ { av1_idct64_new, NULL, NULL }, }; +static INLINE void lowbd_add_flip_buffer_8xn_neon(int16x8_t *in, + uint8_t *output, int stride, + int flipud, + const int height) { + int j = flipud ? (height - 1) : 0; + const int step = flipud ? -1 : 1; + int16x8_t temp_output; + for (int i = 0; i < height; ++i, j += step) { + temp_output = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(output))); + temp_output = vaddq_s16(temp_output, in[j]); + vst1_u8(output, vqmovun_s16(temp_output)); + output += stride; + } +} + +static INLINE uint8x16_t lowbd_get_recon_16x16_neon(const uint8x16_t pred, + int16x8_t res0, + int16x8_t res1) { + int16x8_t temp_output[2]; + uint8x16_t temp_output_8q; + temp_output[0] = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pred))); + temp_output[0] = vaddq_s16(temp_output[0], res0); + temp_output[1] = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pred))); + temp_output[1] = vaddq_s16(temp_output[1], res1); + temp_output_8q = + vcombine_u8(vqmovun_s16(temp_output[0]), vqmovun_s16(temp_output[1])); + return temp_output_8q; +} + +static INLINE void lowbd_add_flip_buffer_16xn_neon(int16x8_t *in, + uint8_t *output, int stride, + int flipud, int height) { + uint8x16_t temp_output_8q; + int j = flipud ? (height - 1) : 0; + const int step = flipud ? -1 : 1; + for (int i = 0; i < height; ++i, j += step) { + temp_output_8q = vld1q_u8(output + i * stride); + temp_output_8q = + lowbd_get_recon_16x16_neon(temp_output_8q, in[j], in[j + height]); + vst1q_u8((output + i * stride), temp_output_8q); + } +} + +static INLINE void dct_const_round_shift_low_8_dual(const int32x4_t *const t32, + int16x8_t *const d0, + int16x8_t *const d1, + int8_t cos_bit) { + const int32x4_t dup_cos_bits_n_32x4 = vdupq_n_s32(-cos_bit); + *d0 = vcombine_s16(vmovn_s32(vrshlq_s32(t32[0], dup_cos_bits_n_32x4)), + vmovn_s32(vrshlq_s32(t32[1], dup_cos_bits_n_32x4))); + *d1 = vcombine_s16(vmovn_s32(vrshlq_s32(t32[2], dup_cos_bits_n_32x4)), + vmovn_s32(vrshlq_s32(t32[3], dup_cos_bits_n_32x4))); +} + +static INLINE void btf_16_lane_0_1_neon(const int16x8_t in0, + const int16x8_t in1, const int16x4_t c, + int16x8_t *t0, int16x8_t *t1) { + int32x4_t s0[2], s1[2]; + int16x4_t v0[2], v1[2]; + + s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 0); + s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 0); + s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 1); + s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 1); + + s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 1); + s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 1); + s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 0); + s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 0); + + v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); + v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); + v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); + v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); + + *t0 = vcombine_s16(v0[0], v0[1]); + *t1 = vcombine_s16(v1[0], v1[1]); +} + +static INLINE void btf_16_lane_2_3_neon(const int16x8_t in0, + const int16x8_t in1, const int16x4_t c, + int16x8_t *t0, int16x8_t *t1) { + int32x4_t s0[2], s1[2]; + int16x4_t v0[2], v1[2]; + + s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); + s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); + s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); + s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); + + s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 3); + s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 3); + s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 2); + s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 2); + + v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); + v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); + v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); + v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); + + *t0 = vcombine_s16(v0[0], v0[1]); + *t1 = vcombine_s16(v1[0], v1[1]); +} + +static INLINE void btf_16_neon(const int16x8_t in0, int16_t coef1, + int16_t coef2, int16x8_t *t0, int16x8_t *t1) { + int32x4_t s0_l, s0_h, s1_l, s1_h; + int16x4_t v0[2], v1[2]; + + s0_l = vmull_n_s16(vget_low_s16(in0), coef1); + s0_h = vmull_n_s16(vget_high_s16(in0), coef1); + s1_l = vmull_n_s16(vget_low_s16(in0), -coef2); + s1_h = vmull_n_s16(vget_high_s16(in0), -coef2); + + v0[0] = vrshrn_n_s32(s0_l, INV_COS_BIT); + v0[1] = vrshrn_n_s32(s0_h, INV_COS_BIT); + v1[0] = vrshrn_n_s32(s1_l, INV_COS_BIT); + v1[1] = vrshrn_n_s32(s1_h, INV_COS_BIT); + + *t0 = vcombine_s16(v0[0], v0[1]); + *t1 = vcombine_s16(v1[0], v1[1]); +} + +static INLINE void btf_16_lane_3_2_neon(const int16x8_t in0, + const int16x8_t in1, const int16x4_t c, + int16x8_t *t0, int16x8_t *t1) { + int32x4_t s0[2], s1[2]; + int16x4_t v0[2], v1[2]; + + s0[0] = vmull_lane_s16(vget_low_s16(in0), c, 3); + s0[1] = vmull_lane_s16(vget_high_s16(in0), c, 3); + s1[0] = vmull_lane_s16(vget_low_s16(in0), c, 2); + s1[1] = vmull_lane_s16(vget_high_s16(in0), c, 2); + + s0[0] = vmlal_lane_s16(s0[0], vget_low_s16(in1), c, 2); + s0[1] = vmlal_lane_s16(s0[1], vget_high_s16(in1), c, 2); + s1[0] = vmlsl_lane_s16(s1[0], vget_low_s16(in1), c, 3); + s1[1] = vmlsl_lane_s16(s1[1], vget_high_s16(in1), c, 3); + + v0[0] = vrshrn_n_s32(s0[0], INV_COS_BIT); + v0[1] = vrshrn_n_s32(s0[1], INV_COS_BIT); + v1[0] = vrshrn_n_s32(s1[0], INV_COS_BIT); + v1[1] = vrshrn_n_s32(s1[1], INV_COS_BIT); + + *t0 = vcombine_s16(v0[0], v0[1]); + *t1 = vcombine_s16(v1[0], v1[1]); +} + +static INLINE void btf_16_half_neon(int16x8_t *const x, const int16x4_t c) { + int32x4_t t0[2], t1[2]; + int16x4_t v0[2], v1[2]; + + // Don't add/sub before multiply, which will overflow in iadst8. + const int32x4_t x0_lo = vmull_lane_s16(vget_low_s16(x[0]), c, 0); + const int32x4_t x0_hi = vmull_lane_s16(vget_high_s16(x[0]), c, 0); + const int32x4_t x1_lo = vmull_lane_s16(vget_low_s16(x[1]), c, 0); + const int32x4_t x1_hi = vmull_lane_s16(vget_high_s16(x[1]), c, 0); + + t0[0] = vaddq_s32(x0_lo, x1_lo); + t0[1] = vaddq_s32(x0_hi, x1_hi); + t1[0] = vsubq_s32(x0_lo, x1_lo); + t1[1] = vsubq_s32(x0_hi, x1_hi); + + v0[0] = vrshrn_n_s32(t0[0], INV_COS_BIT); + v0[1] = vrshrn_n_s32(t0[1], INV_COS_BIT); + v1[0] = vrshrn_n_s32(t1[0], INV_COS_BIT); + v1[1] = vrshrn_n_s32(t1[1], INV_COS_BIT); + + x[0] = vcombine_s16(v0[0], v0[1]); + x[1] = vcombine_s16(v1[0], v1[1]); +} + +static INLINE int16x4_t create_s16x4_neon(int16_t *const c0, int16_t *const c1, + int16_t *const c2, + int16_t *const c3) { + int16x4_t val = vdup_n_s16((int16_t)0); + val = vld1_lane_s16(c0, val, 0); + val = vld1_lane_s16(c1, val, 1); + val = vld1_lane_s16(c2, val, 2); + val = vld1_lane_s16(c3, val, 3); + return val; +} + +static INLINE void iadst8_new_neon(int16x8_t *const in, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + + const int16x4_t c0 = + create_s16x4_neon((int16_t *)(cospi + 4), (int16_t *)(cospi + 60), + (int16_t *)(cospi + 20), (int16_t *)(cospi + 44)); + const int16x4_t c1 = + create_s16x4_neon((int16_t *)(cospi + 36), (int16_t *)(cospi + 28), + (int16_t *)(cospi + 52), (int16_t *)(cospi + 12)); + const int16x4_t c2 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + int16x8_t x[8]; + int16x8_t s0, s1, s2, s3, s4, s5, s6, s7; + + // Stage 1 + x[0] = in[7]; + x[1] = in[0]; + x[2] = in[5]; + x[3] = in[2]; + x[4] = in[3]; + x[5] = in[4]; + x[6] = in[1]; + x[7] = in[6]; + + // Stage 2 + btf_16_lane_0_1_neon(x[0], x[1], c0, &s0, &s1); + btf_16_lane_2_3_neon(x[2], x[3], c0, &s2, &s3); + btf_16_lane_0_1_neon(x[4], x[5], c1, &s4, &s5); + btf_16_lane_2_3_neon(x[6], x[7], c1, &s6, &s7); + + // Stage 3 + x[0] = vqaddq_s16(s0, s4); + x[1] = vqaddq_s16(s1, s5); + x[2] = vqaddq_s16(s2, s6); + x[3] = vqaddq_s16(s3, s7); + x[4] = vqsubq_s16(s0, s4); + x[5] = vqsubq_s16(s1, s5); + x[6] = vqsubq_s16(s2, s6); + x[7] = vqsubq_s16(s3, s7); + + // Stage 4 + s0 = x[0]; + s1 = x[1]; + s2 = x[2]; + s3 = x[3]; + btf_16_lane_2_3_neon(x[4], x[5], c2, &s4, &s5); + btf_16_lane_3_2_neon(x[7], x[6], c2, &s7, &s6); + + // Stage 5 + x[0] = vqaddq_s16(s0, s2); + x[1] = vqaddq_s16(s1, s3); + x[2] = vqsubq_s16(s0, s2); + x[3] = vqsubq_s16(s1, s3); + x[4] = vqaddq_s16(s4, s6); + x[5] = vqaddq_s16(s5, s7); + x[6] = vqsubq_s16(s4, s6); + x[7] = vqsubq_s16(s5, s7); + + // stage 6 + btf_16_half_neon(x + 2, c2); + btf_16_half_neon(x + 6, c2); + + // Stage 7 + out[0] = x[0]; + out[1] = vnegq_s16(x[4]); + out[2] = x[6]; + out[3] = vnegq_s16(x[2]); + out[4] = x[3]; + out[5] = vnegq_s16(x[7]); + out[6] = x[5]; + out[7] = vnegq_s16(x[1]); +} + +static INLINE void iadst8_low1_new_neon(int16x8_t *const in, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + const int16x4_t c2 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + int16x8_t x[8]; + int16x8_t s0, s1, s4, s5; + + // Stage 1 + x[1] = in[0]; + + // Stage 2 + + btf_16_neon(x[1], cospi[60], cospi[4], &s0, &s1); + + // Stage 3 + x[0] = s0; + x[1] = s1; + x[4] = s0; + x[5] = s1; + + // Stage 4 + s0 = x[0]; + s1 = x[1]; + btf_16_lane_2_3_neon(x[4], x[5], c2, &s4, &s5); + + // Stage 5 + x[0] = s0; + x[1] = s1; + x[2] = s0; + x[3] = s1; + x[4] = s4; + x[5] = s5; + x[6] = s4; + x[7] = s5; + + // stage 6 + btf_16_half_neon(x + 2, c2); + btf_16_half_neon(x + 6, c2); + + // Stage 7 + out[0] = x[0]; + out[1] = vnegq_s16(x[4]); + out[2] = x[6]; + out[3] = vnegq_s16(x[2]); + out[4] = x[3]; + out[5] = vnegq_s16(x[7]); + out[6] = x[5]; + out[7] = vnegq_s16(x[1]); +} + +static INLINE void idct8_new_neon(int16x8_t *in, int16x8_t *out, int8_t cos_bit, + int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + int16x8_t step1[8], step2[8]; + const int16x4_t c0 = + create_s16x4_neon((int16_t *)(cospi + 8), (int16_t *)(cospi + 56), + (int16_t *)(cospi + 40), (int16_t *)(cospi + 24)); + const int16x4_t c2 = + create_s16x4_neon((int16_t *)(cospi + 32), (int16_t *)(cospi + 32), + (int16_t *)(cospi + 16), (int16_t *)(cospi + 48)); + + // stage 2 + btf_16_lane_0_1_neon(in[1], in[7], c0, &step1[7], &step1[4]); + btf_16_lane_2_3_neon(in[5], in[3], c0, &step1[6], &step1[5]); + + // stage 3 + btf_16_lane_0_1_neon(in[0], in[4], c2, &step2[0], &step2[1]); + btf_16_lane_2_3_neon(in[2], in[6], c2, &step2[3], &step2[2]); + step2[4] = vqaddq_s16(step1[4], step1[5]); + step2[5] = vqsubq_s16(step1[4], step1[5]); + step2[6] = vqsubq_s16(step1[7], step1[6]); + step2[7] = vqaddq_s16(step1[7], step1[6]); + + // stage 4 + step1[0] = vqaddq_s16(step2[0], step2[3]); + step1[1] = vqaddq_s16(step2[1], step2[2]); + step1[2] = vqsubq_s16(step2[1], step2[2]); + step1[3] = vqsubq_s16(step2[0], step2[3]); + btf_16_lane_0_1_neon(step2[6], step2[5], c2, &step1[6], &step1[5]); + + // stage 5 + out[0] = vqaddq_s16(step1[0], step2[7]); + out[1] = vqaddq_s16(step1[1], step1[6]); + out[2] = vqaddq_s16(step1[2], step1[5]); + out[3] = vqaddq_s16(step1[3], step2[4]); + out[4] = vqsubq_s16(step1[3], step2[4]); + out[5] = vqsubq_s16(step1[2], step1[5]); + out[6] = vqsubq_s16(step1[1], step1[6]); + out[7] = vqsubq_s16(step1[0], step2[7]); +} + +static INLINE void idct8_low1_new_neon(int16x8_t *in, int16x8_t *out, + int8_t cos_bit, int bit) { + (void)bit; + const int32_t *cospi = cospi_arr(cos_bit); + int16x4_t step1l[4], step1h[4]; + int16x8_t step1[8], step2[8]; + int32x4_t t32[8]; + + // stage 1 + step1l[0] = vget_low_s16(in[0]); + step1h[0] = vget_high_s16(in[0]); + + // stage 2 + t32[2] = vmull_n_s16(step1l[0], (int16_t)cospi[32]); + t32[3] = vmull_n_s16(step1h[0], (int16_t)cospi[32]); + + t32[0] = t32[2]; + t32[1] = t32[3]; + dct_const_round_shift_low_8_dual(&t32[0], &step2[0], &step2[1], cos_bit); + + // stage 3 + step1[0] = step2[0]; + step1[1] = step2[1]; + step1[2] = step2[1]; + step1[3] = step2[0]; + + // stage 4 + out[0] = step1[0]; + out[1] = step1[1]; + out[2] = step1[2]; + out[3] = step1[3]; + out[4] = step1[3]; + out[5] = step1[2]; + out[6] = step1[1]; + out[7] = step1[0]; +} + +void av1_round_shift_array_16_neon(int16x8_t *arr, int size, int bit) { + assert(!(size % 4)); + if (!bit) return; + const int16x8_t dup_bits_n_16x8 = vdupq_n_s16((int16_t)(-bit)); + for (int i = 0; i < size; i++) { + arr[i] = vrshlq_s16(arr[i], dup_bits_n_16x8); + } +} + +static INLINE void flip_buf_ud_neon(int16x8_t *input, int size) { + int16x8_t temp[8]; + for (int i = 0; i < size; ++i) { + temp[i] = input[size - 1 - i]; + } + for (int i = 0; i < size; ++i) { + input[i] = temp[i]; + } +} + +static INLINE void load_buffer_32bit_to_16bit_neon(const int32_t *input, + int16x8_t *const a, + int out_size) { + for (int i = 0; i < 8; ++i) { + a[i] = vcombine_s16(vmovn_s32(vld1q_s32(input)), + vmovn_s32(vld1q_s32(input + 4))); + input += out_size; + } +} + +static INLINE void av1_identity8_new_neon(int16x8_t *input, int16x8_t *output, + int8_t cos_bit, int bit) { + (void)bit; + (void)cos_bit; + + output[0] = vmulq_n_s16(input[0], (int16_t)2); + output[1] = vmulq_n_s16(input[1], (int16_t)2); + output[2] = vmulq_n_s16(input[2], (int16_t)2); + output[3] = vmulq_n_s16(input[3], (int16_t)2); + output[4] = vmulq_n_s16(input[4], (int16_t)2); + output[5] = vmulq_n_s16(input[5], (int16_t)2); + output[6] = vmulq_n_s16(input[6], (int16_t)2); + output[7] = vmulq_n_s16(input[7], (int16_t)2); +} + // Functions for blocks with eob at DC and within // topleft 8x8, 16x16, 32x32 corner static const transform_1d_neon @@ -90,10 +530,33 @@ { NULL, NULL, NULL, NULL }, { NULL, NULL, NULL, NULL } } }; -static INLINE void lowbd_inv_txfm2d_add_idtx_neon(const int32_t *input, - uint8_t *output, int stride, - TX_TYPE tx_type, - TX_SIZE tx_size, int eob) { + +static const transform_neon + lowbd_txfm_all_1d_zeros_w_arr[TX_SIZES][ITX_TYPES_1D][4] = { + { + { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL }, + }, + { { idct8_low1_new_neon, idct8_new_neon, NULL, NULL }, + { iadst8_low1_new_neon, iadst8_new_neon, NULL, NULL }, + { av1_identity8_new_neon, av1_identity8_new_neon, NULL, NULL } }, + { + { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL }, + }, + { { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL } }, + { { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL }, + { NULL, NULL, NULL, NULL } } + }; + +static INLINE void lowbd_inv_txfm2d_add_wxh_idtx_neon( + const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob) { DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); int32_t *temp_in = txfm_buf; @@ -160,7 +623,62 @@ } } -static INLINE void lowbd_inv_txfm2d_add_v_identity_neon( +static INLINE void lowbd_inv_txfm2d_add_idtx_neon(const int32_t *input, + uint8_t *output, int stride, + TX_TYPE tx_type, + TX_SIZE tx_size, int eob) { + int16x8_t a[64 * 8]; + int eobx, eoby; + get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; + const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int buf_size_w_div8 = txfm_size_col >> 3; + const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; + const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; + const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; + const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; + const int32_t *input_1; + const transform_neon row_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; + const transform_neon col_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + + for (int i = 0; i < buf_size_nonzero_h_div8; i++) { + input_1 = input; + for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); + input_1 += 8; + } + input += (txfm_size_col * 8); + row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); + av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, + -shift[0]); + } + for (int j = 0; j < buf_size_w_div8; ++j) { + col_txfm(&a[j * txfm_size_row], &a[j * txfm_size_row], cos_bit_col, 0); + av1_round_shift_array_16_neon(&a[j * txfm_size_row], txfm_size_row, + -shift[1]); + } + if (txfm_size_col >= 16) { + for (int i = 0; i < (txfm_size_col >> 4); i++) { + lowbd_add_flip_buffer_16xn_neon( + &a[i * txfm_size_row * 2], output + 16 * i, stride, 0, txfm_size_row); + } + } else if (txfm_size_col == 8) { + lowbd_add_flip_buffer_8xn_neon(a, output, stride, 0, txfm_size_row); + } +} + +static INLINE void lowbd_inv_txfm2d_add_v_wxh_identity_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); @@ -244,7 +762,81 @@ } } -static INLINE void lowbd_inv_txfm2d_add_h_identity_neon( +static INLINE void lowbd_inv_txfm2d_add_v_identity_neon( + const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob) { + int16x8_t a[64 * 8]; + int16x8_t b[64 * 8]; + int eobx, eoby, ud_flip, lr_flip; + get_eobx_eoby_scan_v_identity(&eobx, &eoby, tx_size, eob); + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; + const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int buf_size_w_div8 = txfm_size_col >> 3; + const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; + const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; + const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; + const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; + const int32_t *input_1; + int temp_b = 0; + const transform_neon row_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; + const transform_neon col_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + for (int i = 0; i < buf_size_nonzero_h_div8; i++) { + input_1 = input; + for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); + transpose_s16_8x8q(&a[k], &a[k]); + input_1 += 8; + } + input += (txfm_size_col * 8); + row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); + av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, + -shift[0]); + if (lr_flip == 1) { + for (int j = 0; j < buf_size_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + flip_buf_ud_neon(&a[k], 8); + transpose_s16_8x8q( + &a[k], &b[temp_b + txfm_size_row * (buf_size_w_div8 - 1 - j)]); + } + temp_b += 8; + } else { + for (int j = 0; j < buf_size_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + transpose_s16_8x8q(&a[k], &b[temp_b + txfm_size_row * j]); + } + temp_b += 8; + } + } + for (int j = 0; j < buf_size_w_div8; ++j) { + col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], cos_bit_col, 0); + av1_round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row, + -shift[1]); + } + if (txfm_size_col >= 16) { + for (int i = 0; i < (txfm_size_col >> 4); i++) { + lowbd_add_flip_buffer_16xn_neon( + &b[i * txfm_size_row * 2], output + 16 * i, stride, 0, txfm_size_row); + } + } else if (txfm_size_col == 8) { + lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, txfm_size_row); + } +} + +static INLINE void lowbd_inv_txfm2d_add_h_wxh_identity_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { DECLARE_ALIGNED(32, int, txfm_buf[32 * 32 + 32 + 32]); @@ -328,6 +920,63 @@ } } +static INLINE void lowbd_inv_txfm2d_add_h_identity_neon( + const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob) { + int16x8_t a[64 * 8]; + int eobx, eoby, ud_flip, lr_flip; + get_eobx_eoby_scan_h_identity(&eobx, &eoby, tx_size, eob); + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; + const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int buf_size_w_div8 = txfm_size_col >> 3; + const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; + const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; + const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; + const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; + const int32_t *input_1; + const transform_neon row_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; + const transform_neon col_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + for (int i = 0; i < buf_size_nonzero_h_div8; i++) { + input_1 = input; + for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); + input_1 += 8; + } + input += (txfm_size_col * 8); + row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); + av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, + -shift[0]); + } + for (int j = 0; j < buf_size_w_div8; ++j) { + col_txfm(&a[j * txfm_size_row], &a[j * txfm_size_row], cos_bit_col, 0); + av1_round_shift_array_16_neon(&a[j * txfm_size_row], txfm_size_row, + -shift[1]); + } + if (txfm_size_col >= 16) { + for (int i = 0; i < (txfm_size_col >> 4); i++) { + lowbd_add_flip_buffer_16xn_neon(&a[i * txfm_size_row * 2], + output + 16 * i, stride, ud_flip, + txfm_size_row); + } + } else if (txfm_size_col == 8) { + lowbd_add_flip_buffer_8xn_neon(a, output, stride, ud_flip, txfm_size_row); + } +} + static INLINE void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, @@ -644,7 +1293,7 @@ } } -static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( +static INLINE void lowbd_inv_txfm2d_add_wxh_no_identity_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { DECLARE_ALIGNED(32, int, txfm_buf[64 * 64 + 64 + 64]); @@ -727,6 +1376,113 @@ } } +static INLINE void lowbd_inv_txfm2d_add_no_identity_neon( + const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob) { + int16x8_t a[64 * 8]; + int16x8_t b[64 * 8]; + int eobx, eoby, ud_flip, lr_flip; + get_eobx_eoby_scan_default(&eobx, &eoby, tx_size, eob); + const int8_t *shift = inv_txfm_shift_ls[tx_size]; + const int txw_idx = get_txw_idx(tx_size); + const int txh_idx = get_txh_idx(tx_size); + const int cos_bit_col = inv_cos_bit_col[txw_idx][txh_idx]; + const int cos_bit_row = inv_cos_bit_row[txw_idx][txh_idx]; + const int txfm_size_col = tx_size_wide[tx_size]; + const int txfm_size_row = tx_size_high[tx_size]; + const int buf_size_w_div8 = txfm_size_col >> 3; + const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3; + const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3; + const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx]; + const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby]; + const int32_t *input_1; + int temp_b = 0; + + const transform_neon row_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x]; + const transform_neon col_txfm = + lowbd_txfm_all_1d_zeros_w_arr[txh_idx][vitx_1d_tab[tx_type]][fun_idx_y]; + + assert(col_txfm != NULL); + assert(row_txfm != NULL); + + get_flip_cfg(tx_type, &ud_flip, &lr_flip); + + for (int i = 0; i < buf_size_nonzero_h_div8; i++) { + input_1 = input; + for (int j = 0; j < buf_size_nonzero_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + load_buffer_32bit_to_16bit_neon(input_1, &a[k], txfm_size_col); + transpose_s16_8x8q(&a[k], &a[k]); + input_1 += 8; + } + input += (txfm_size_col * 8); + row_txfm(&a[i * txfm_size_col], &a[i * txfm_size_col], cos_bit_row, 0); + av1_round_shift_array_16_neon(&a[i * txfm_size_col], txfm_size_col, + -shift[0]); + if (lr_flip == 1) { + for (int j = 0; j < buf_size_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + flip_buf_ud_neon(&a[k], 8); + transpose_s16_8x8q( + &a[k], &b[temp_b + txfm_size_row * (buf_size_w_div8 - 1 - j)]); + } + temp_b += 8; + } else { + for (int j = 0; j < buf_size_w_div8; ++j) { + int k = j * 8 + i * txfm_size_col; + transpose_s16_8x8q(&a[k], &b[temp_b + txfm_size_row * j]); + } + temp_b += 8; + } + } + for (int j = 0; j < buf_size_w_div8; ++j) { + col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], cos_bit_col, 0); + av1_round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row, + -shift[1]); + } + + if (txfm_size_col >= 16) { + for (int i = 0; i < (txfm_size_col >> 4); i++) { + lowbd_add_flip_buffer_16xn_neon(&b[i * txfm_size_row * 2], + output + 16 * i, stride, ud_flip, + txfm_size_row); + } + } else if (txfm_size_col == 8) { + lowbd_add_flip_buffer_8xn_neon(b, output, stride, ud_flip, txfm_size_row); + } +} + +static INLINE void lowbd_inv_txfm2d_add_wxh_universe_neon( + const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, + TX_SIZE tx_size, int eob) { + switch (tx_type) { + case IDTX: + lowbd_inv_txfm2d_add_wxh_idtx_neon(input, output, stride, tx_type, + tx_size, eob); + break; + + case H_DCT: + case H_ADST: + case H_FLIPADST: + lowbd_inv_txfm2d_add_v_wxh_identity_neon(input, output, stride, tx_type, + tx_size, eob); + break; + + case V_DCT: + case V_ADST: + case V_FLIPADST: + lowbd_inv_txfm2d_add_h_wxh_identity_neon(input, output, stride, tx_type, + tx_size, eob); + break; + + default: + lowbd_inv_txfm2d_add_wxh_no_identity_neon(input, output, stride, tx_type, + tx_size, eob); + break; + } +} + static INLINE void lowbd_inv_txfm2d_add_universe_neon( const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { @@ -756,6 +1512,7 @@ break; } } + void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type, TX_SIZE tx_size, int eob) { @@ -787,8 +1544,8 @@ break; case TX_16X64: { - lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type, - tx_size, eob); + lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, + tx_size, eob); } break; case TX_64X16: { @@ -797,13 +1554,13 @@ memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); } - lowbd_inv_txfm2d_add_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); + lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, + tx_size, eob); } break; case TX_32X64: { - lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type, - tx_size, eob); + lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, + tx_size, eob); } break; case TX_64X32: { @@ -812,8 +1569,8 @@ memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); } - lowbd_inv_txfm2d_add_universe_neon(mod_input, output, stride, tx_type, - tx_size, eob); + lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, + tx_size, eob); } break; case TX_64X64: { @@ -822,13 +1579,18 @@ memcpy(mod_input + row * 64, input + row * 32, 32 * sizeof(*mod_input)); memset(mod_input + row * 64 + 32, 0, 32 * sizeof(*mod_input)); } - lowbd_inv_txfm2d_add_universe_neon(mod_input, output, stride, tx_type, + lowbd_inv_txfm2d_add_wxh_universe_neon(mod_input, output, stride, tx_type, + tx_size, eob); + } break; + + case TX_8X8: { + lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type, tx_size, eob); } break; default: - lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type, - tx_size, eob); + lowbd_inv_txfm2d_add_wxh_universe_neon(input, output, stride, tx_type, + tx_size, eob); break; } }
diff --git a/av1/common/arm/av1_inv_txfm_neon.h b/av1/common/arm/av1_inv_txfm_neon.h index 6af2d61..7f1e9fe 100644 --- a/av1/common/arm/av1_inv_txfm_neon.h +++ b/av1/common/arm/av1_inv_txfm_neon.h
@@ -23,6 +23,8 @@ typedef void (*transform_1d_neon)(const int32_t *input, int32_t *output, const int8_t cos_bit, const int8_t *stage_ptr); +typedef void (*transform_neon)(int16x8_t *input, int16x8_t *output, + int8_t cos_bit, int bit); DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_8x8_default[8]) = { 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707,
diff --git a/av1/common/arm/transpose_neon.h b/av1/common/arm/transpose_neon.h index fe13408..c4a414b 100644 --- a/av1/common/arm/transpose_neon.h +++ b/av1/common/arm/transpose_neon.h
@@ -386,6 +386,83 @@ vget_high_s16(vreinterpretq_s16_s32(c3.val[1]))); } +static INLINE int16x8x2_t vpx_vtrnq_s64_to_s16(int32x4_t a0, int32x4_t a1) { + int16x8x2_t b0; + b0.val[0] = vcombine_s16(vreinterpret_s16_s32(vget_low_s32(a0)), + vreinterpret_s16_s32(vget_low_s32(a1))); + b0.val[1] = vcombine_s16(vreinterpret_s16_s32(vget_high_s32(a0)), + vreinterpret_s16_s32(vget_high_s32(a1))); + return b0; +} + +static INLINE void transpose_s16_8x8q(int16x8_t *a0, int16x8_t *out) { + // Swap 16 bit elements. Goes from: + // a0: 00 01 02 03 04 05 06 07 + // a1: 10 11 12 13 14 15 16 17 + // a2: 20 21 22 23 24 25 26 27 + // a3: 30 31 32 33 34 35 36 37 + // a4: 40 41 42 43 44 45 46 47 + // a5: 50 51 52 53 54 55 56 57 + // a6: 60 61 62 63 64 65 66 67 + // a7: 70 71 72 73 74 75 76 77 + // to: + // b0.val[0]: 00 10 02 12 04 14 06 16 + // b0.val[1]: 01 11 03 13 05 15 07 17 + // b1.val[0]: 20 30 22 32 24 34 26 36 + // b1.val[1]: 21 31 23 33 25 35 27 37 + // b2.val[0]: 40 50 42 52 44 54 46 56 + // b2.val[1]: 41 51 43 53 45 55 47 57 + // b3.val[0]: 60 70 62 72 64 74 66 76 + // b3.val[1]: 61 71 63 73 65 75 67 77 + + const int16x8x2_t b0 = vtrnq_s16(*a0, *(a0 + 1)); + const int16x8x2_t b1 = vtrnq_s16(*(a0 + 2), *(a0 + 3)); + const int16x8x2_t b2 = vtrnq_s16(*(a0 + 4), *(a0 + 5)); + const int16x8x2_t b3 = vtrnq_s16(*(a0 + 6), *(a0 + 7)); + + // Swap 32 bit elements resulting in: + // c0.val[0]: 00 10 20 30 04 14 24 34 + // c0.val[1]: 02 12 22 32 06 16 26 36 + // c1.val[0]: 01 11 21 31 05 15 25 35 + // c1.val[1]: 03 13 23 33 07 17 27 37 + // c2.val[0]: 40 50 60 70 44 54 64 74 + // c2.val[1]: 42 52 62 72 46 56 66 76 + // c3.val[0]: 41 51 61 71 45 55 65 75 + // c3.val[1]: 43 53 63 73 47 57 67 77 + + const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]), + vreinterpretq_s32_s16(b1.val[0])); + const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]), + vreinterpretq_s32_s16(b1.val[1])); + const int32x4x2_t c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]), + vreinterpretq_s32_s16(b3.val[0])); + const int32x4x2_t c3 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[1]), + vreinterpretq_s32_s16(b3.val[1])); + + // Swap 64 bit elements resulting in: + // d0.val[0]: 00 10 20 30 40 50 60 70 + // d0.val[1]: 04 14 24 34 44 54 64 74 + // d1.val[0]: 01 11 21 31 41 51 61 71 + // d1.val[1]: 05 15 25 35 45 55 65 75 + // d2.val[0]: 02 12 22 32 42 52 62 72 + // d2.val[1]: 06 16 26 36 46 56 66 76 + // d3.val[0]: 03 13 23 33 43 53 63 73 + // d3.val[1]: 07 17 27 37 47 57 67 77 + const int16x8x2_t d0 = vpx_vtrnq_s64_to_s16(c0.val[0], c2.val[0]); + const int16x8x2_t d1 = vpx_vtrnq_s64_to_s16(c1.val[0], c3.val[0]); + const int16x8x2_t d2 = vpx_vtrnq_s64_to_s16(c0.val[1], c2.val[1]); + const int16x8x2_t d3 = vpx_vtrnq_s64_to_s16(c1.val[1], c3.val[1]); + + *out = d0.val[0]; + *(out + 1) = d1.val[0]; + *(out + 2) = d2.val[0]; + *(out + 3) = d3.val[0]; + *(out + 4) = d0.val[1]; + *(out + 5) = d1.val[1]; + *(out + 6) = d2.val[1]; + *(out + 7) = d3.val[1]; +} + static INLINE void transpose_s16_4x4d(int16x4_t *a0, int16x4_t *a1, int16x4_t *a2, int16x4_t *a3) { // Swap 16 bit elements. Goes from: