Google Git Google Git
Sign in
aomedia/avm/8acbd506b0dd5b3f7e4e88837b151c830c6282d9/./aom_dsp/x86/aom_subpixel_8t_sse2.asm
blob: 1b5c7bb6eb4d74b6e5a0489049b314f659188f50 [file]
;
; Copyright (c) 2021, Alliance for Open Media. All rights reserved
;
; This source code is subject to the terms of the BSD 3-Clause Clear License and the
; Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear License was
; not distributed with this source code in the LICENSE file, you can obtain it
; at aomedia.org/license/software-license/bsd-3-c-c/. If the Alliance for Open Media Patent
; License 1.0 was not distributed with this source code in the PATENTS file, you
; can obtain it at aomedia.org/license/patent-license/.
;
;
%include "aom_ports/x86_abi_support.asm"
;Note: tap3 and tap4 have to be applied and added after other taps to avoid
;overflow.
%macro GET_FILTERS_4 0
mov rdx, arg(5) ;filter ptr
mov rcx, 0x0400040
movdqa xmm7, [rdx] ;load filters
pshuflw xmm0, xmm7, 0b ;k0
pshuflw xmm1, xmm7, 01010101b ;k1
pshuflw xmm2, xmm7, 10101010b ;k2
pshuflw xmm3, xmm7, 11111111b ;k3
psrldq xmm7, 8
pshuflw xmm4, xmm7, 0b ;k4
pshuflw xmm5, xmm7, 01010101b ;k5
pshuflw xmm6, xmm7, 10101010b ;k6
pshuflw xmm7, xmm7, 11111111b ;k7
punpcklqdq xmm0, xmm1
punpcklqdq xmm2, xmm3
punpcklqdq xmm5, xmm4
punpcklqdq xmm6, xmm7
movdqa k0k1, xmm0
movdqa k2k3, xmm2
movdqa k5k4, xmm5
movdqa k6k7, xmm6
movq xmm6, rcx
pshufd xmm6, xmm6, 0
movdqa krd, xmm6
pxor xmm7, xmm7
movdqa zero, xmm7
%endm
%macro APPLY_FILTER_4 1
punpckldq xmm0, xmm1 ;two row in one register
punpckldq xmm6, xmm7
punpckldq xmm2, xmm3
punpckldq xmm5, xmm4
punpcklbw xmm0, zero ;unpack to word
punpcklbw xmm6, zero
punpcklbw xmm2, zero
punpcklbw xmm5, zero
pmullw xmm0, k0k1 ;multiply the filter factors
pmullw xmm6, k6k7
pmullw xmm2, k2k3
pmullw xmm5, k5k4
paddsw xmm0, xmm6 ;sum
movdqa xmm1, xmm0
psrldq xmm1, 8
paddsw xmm0, xmm1
paddsw xmm0, xmm2
psrldq xmm2, 8
paddsw xmm0, xmm5
psrldq xmm5, 8
paddsw xmm0, xmm2
paddsw xmm0, xmm5
paddsw xmm0, krd ;rounding
psraw xmm0, 7 ;shift
packuswb xmm0, xmm0 ;pack to byte
%if %1
movd xmm1, [rdi]
pavgb xmm0, xmm1
%endif
movd [rdi], xmm0
%endm
%macro GET_FILTERS 0
mov rdx, arg(5) ;filter ptr
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;output_ptr
mov rcx, 0x0400040
movdqa xmm7, [rdx] ;load filters
pshuflw xmm0, xmm7, 0b ;k0
pshuflw xmm1, xmm7, 01010101b ;k1
pshuflw xmm2, xmm7, 10101010b ;k2
pshuflw xmm3, xmm7, 11111111b ;k3
pshufhw xmm4, xmm7, 0b ;k4
pshufhw xmm5, xmm7, 01010101b ;k5
pshufhw xmm6, xmm7, 10101010b ;k6
pshufhw xmm7, xmm7, 11111111b ;k7
punpcklwd xmm0, xmm0
punpcklwd xmm1, xmm1
punpcklwd xmm2, xmm2
punpcklwd xmm3, xmm3
punpckhwd xmm4, xmm4
punpckhwd xmm5, xmm5
punpckhwd xmm6, xmm6
punpckhwd xmm7, xmm7
movdqa k0, xmm0 ;store filter factors on stack
movdqa k1, xmm1
movdqa k2, xmm2
movdqa k3, xmm3
movdqa k4, xmm4
movdqa k5, xmm5
movdqa k6, xmm6
movdqa k7, xmm7
movq xmm6, rcx
pshufd xmm6, xmm6, 0
movdqa krd, xmm6 ;rounding
pxor xmm7, xmm7
movdqa zero, xmm7
%endm
%macro LOAD_VERT_8 1
movq xmm0, [rsi + %1] ;0
movq xmm1, [rsi + rax + %1] ;1
movq xmm6, [rsi + rdx * 2 + %1] ;6
lea rsi, [rsi + rax]
movq xmm7, [rsi + rdx * 2 + %1] ;7
movq xmm2, [rsi + rax + %1] ;2
movq xmm3, [rsi + rax * 2 + %1] ;3
movq xmm4, [rsi + rdx + %1] ;4
movq xmm5, [rsi + rax * 4 + %1] ;5
%endm
%macro APPLY_FILTER_8 2
punpcklbw xmm0, zero
punpcklbw xmm1, zero
punpcklbw xmm6, zero
punpcklbw xmm7, zero
punpcklbw xmm2, zero
punpcklbw xmm5, zero
punpcklbw xmm3, zero
punpcklbw xmm4, zero
pmullw xmm0, k0
pmullw xmm1, k1
pmullw xmm6, k6
pmullw xmm7, k7
pmullw xmm2, k2
pmullw xmm5, k5
pmullw xmm3, k3
pmullw xmm4, k4
paddsw xmm0, xmm1
paddsw xmm0, xmm6
paddsw xmm0, xmm7
paddsw xmm0, xmm2
paddsw xmm0, xmm5
paddsw xmm0, xmm3
paddsw xmm0, xmm4
paddsw xmm0, krd ;rounding
psraw xmm0, 7 ;shift
packuswb xmm0, xmm0 ;pack back to byte
%if %1
movq xmm1, [rdi + %2]
pavgb xmm0, xmm1
%endif
movq [rdi + %2], xmm0
%endm
SECTION .text
;void aom_filter_block1d4_v8_sse2
;(
; unsigned char *src_ptr,
; unsigned int src_pitch,
; unsigned char *output_ptr,
; unsigned int out_pitch,
; unsigned int output_height,
; short *filter
;)
globalsym(aom_filter_block1d4_v8_sse2)
sym(aom_filter_block1d4_v8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 6
SAVE_XMM 7
push rsi
push rdi
push rbx
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 6
%define k0k1 [rsp + 16 * 0]
%define k2k3 [rsp + 16 * 1]
%define k5k4 [rsp + 16 * 2]
%define k6k7 [rsp + 16 * 3]
%define krd [rsp + 16 * 4]
%define zero [rsp + 16 * 5]
GET_FILTERS_4
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;output_ptr
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rbx, DWORD PTR arg(3) ;out_pitch
lea rdx, [rax + rax * 2]
movsxd rcx, DWORD PTR arg(4) ;output_height
.loop:
movd xmm0, [rsi] ;load src: row 0
movd xmm1, [rsi + rax] ;1
movd xmm6, [rsi + rdx * 2] ;6
lea rsi, [rsi + rax]
movd xmm7, [rsi + rdx * 2] ;7
movd xmm2, [rsi + rax] ;2
movd xmm3, [rsi + rax * 2] ;3
movd xmm4, [rsi + rdx] ;4
movd xmm5, [rsi + rax * 4] ;5
APPLY_FILTER_4 0
lea rdi, [rdi + rbx]
dec rcx
jnz .loop
add rsp, 16 * 6
pop rsp
pop rbx
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void aom_filter_block1d8_v8_sse2
;(
; unsigned char *src_ptr,
; unsigned int src_pitch,
; unsigned char *output_ptr,
; unsigned int out_pitch,
; unsigned int output_height,
; short *filter
;)
globalsym(aom_filter_block1d8_v8_sse2)
sym(aom_filter_block1d8_v8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 6
SAVE_XMM 7
push rsi
push rdi
push rbx
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 10
%define k0 [rsp + 16 * 0]
%define k1 [rsp + 16 * 1]
%define k2 [rsp + 16 * 2]
%define k3 [rsp + 16 * 3]
%define k4 [rsp + 16 * 4]
%define k5 [rsp + 16 * 5]
%define k6 [rsp + 16 * 6]
%define k7 [rsp + 16 * 7]
%define krd [rsp + 16 * 8]
%define zero [rsp + 16 * 9]
GET_FILTERS
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rbx, DWORD PTR arg(3) ;out_pitch
lea rdx, [rax + rax * 2]
movsxd rcx, DWORD PTR arg(4) ;output_height
.loop:
LOAD_VERT_8 0
APPLY_FILTER_8 0, 0
lea rdi, [rdi + rbx]
dec rcx
jnz .loop
add rsp, 16 * 10
pop rsp
pop rbx
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void aom_filter_block1d16_v8_sse2
;(
; unsigned char *src_ptr,
; unsigned int src_pitch,
; unsigned char *output_ptr,
; unsigned int out_pitch,
; unsigned int output_height,
; short *filter
;)
globalsym(aom_filter_block1d16_v8_sse2)
sym(aom_filter_block1d16_v8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 6
SAVE_XMM 7
push rsi
push rdi
push rbx
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 10
%define k0 [rsp + 16 * 0]
%define k1 [rsp + 16 * 1]
%define k2 [rsp + 16 * 2]
%define k3 [rsp + 16 * 3]
%define k4 [rsp + 16 * 4]
%define k5 [rsp + 16 * 5]
%define k6 [rsp + 16 * 6]
%define k7 [rsp + 16 * 7]
%define krd [rsp + 16 * 8]
%define zero [rsp + 16 * 9]
GET_FILTERS
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rbx, DWORD PTR arg(3) ;out_pitch
lea rdx, [rax + rax * 2]
movsxd rcx, DWORD PTR arg(4) ;output_height
.loop:
LOAD_VERT_8 0
APPLY_FILTER_8 0, 0
sub rsi, rax
LOAD_VERT_8 8
APPLY_FILTER_8 0, 8
add rdi, rbx
dec rcx
jnz .loop
add rsp, 16 * 10
pop rsp
pop rbx
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void aom_filter_block1d4_h8_sse2
;(
; unsigned char *src_ptr,
; unsigned int src_pixels_per_line,
; unsigned char *output_ptr,
; unsigned int output_pitch,
; unsigned int output_height,
; short *filter
;)
globalsym(aom_filter_block1d4_h8_sse2)
sym(aom_filter_block1d4_h8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 6
SAVE_XMM 7
push rsi
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 6
%define k0k1 [rsp + 16 * 0]
%define k2k3 [rsp + 16 * 1]
%define k5k4 [rsp + 16 * 2]
%define k6k7 [rsp + 16 * 3]
%define krd [rsp + 16 * 4]
%define zero [rsp + 16 * 5]
GET_FILTERS_4
mov rsi, arg(0) ;src_ptr
mov rdi, arg(2) ;output_ptr
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rdx, DWORD PTR arg(3) ;out_pitch
movsxd rcx, DWORD PTR arg(4) ;output_height
.loop:
movdqu xmm0, [rsi - 3] ;load src
movdqa xmm1, xmm0
movdqa xmm6, xmm0
movdqa xmm7, xmm0
movdqa xmm2, xmm0
movdqa xmm3, xmm0
movdqa xmm5, xmm0
movdqa xmm4, xmm0
psrldq xmm1, 1
psrldq xmm6, 6
psrldq xmm7, 7
psrldq xmm2, 2
psrldq xmm3, 3
psrldq xmm5, 5
psrldq xmm4, 4
APPLY_FILTER_4 0
lea rsi, [rsi + rax]
lea rdi, [rdi + rdx]
dec rcx
jnz .loop
add rsp, 16 * 6
pop rsp
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void aom_filter_block1d8_h8_sse2
;(
; unsigned char *src_ptr,
; unsigned int src_pixels_per_line,
; unsigned char *output_ptr,
; unsigned int output_pitch,
; unsigned int output_height,
; short *filter
;)
globalsym(aom_filter_block1d8_h8_sse2)
sym(aom_filter_block1d8_h8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 6
SAVE_XMM 7
push rsi
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 10
%define k0 [rsp + 16 * 0]
%define k1 [rsp + 16 * 1]
%define k2 [rsp + 16 * 2]
%define k3 [rsp + 16 * 3]
%define k4 [rsp + 16 * 4]
%define k5 [rsp + 16 * 5]
%define k6 [rsp + 16 * 6]
%define k7 [rsp + 16 * 7]
%define krd [rsp + 16 * 8]
%define zero [rsp + 16 * 9]
GET_FILTERS
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rdx, DWORD PTR arg(3) ;out_pitch
movsxd rcx, DWORD PTR arg(4) ;output_height
.loop:
movdqu xmm0, [rsi - 3] ;load src
movdqa xmm1, xmm0
movdqa xmm6, xmm0
movdqa xmm7, xmm0
movdqa xmm2, xmm0
movdqa xmm5, xmm0
movdqa xmm3, xmm0
movdqa xmm4, xmm0
psrldq xmm1, 1
psrldq xmm6, 6
psrldq xmm7, 7
psrldq xmm2, 2
psrldq xmm5, 5
psrldq xmm3, 3
psrldq xmm4, 4
APPLY_FILTER_8 0, 0
lea rsi, [rsi + rax]
lea rdi, [rdi + rdx]
dec rcx
jnz .loop
add rsp, 16 * 10
pop rsp
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret
;void aom_filter_block1d16_h8_sse2
;(
; unsigned char *src_ptr,
; unsigned int src_pixels_per_line,
; unsigned char *output_ptr,
; unsigned int output_pitch,
; unsigned int output_height,
; short *filter
;)
globalsym(aom_filter_block1d16_h8_sse2)
sym(aom_filter_block1d16_h8_sse2):
push rbp
mov rbp, rsp
SHADOW_ARGS_TO_STACK 6
SAVE_XMM 7
push rsi
push rdi
; end prolog
ALIGN_STACK 16, rax
sub rsp, 16 * 10
%define k0 [rsp + 16 * 0]
%define k1 [rsp + 16 * 1]
%define k2 [rsp + 16 * 2]
%define k3 [rsp + 16 * 3]
%define k4 [rsp + 16 * 4]
%define k5 [rsp + 16 * 5]
%define k6 [rsp + 16 * 6]
%define k7 [rsp + 16 * 7]
%define krd [rsp + 16 * 8]
%define zero [rsp + 16 * 9]
GET_FILTERS
movsxd rax, DWORD PTR arg(1) ;pixels_per_line
movsxd rdx, DWORD PTR arg(3) ;out_pitch
movsxd rcx, DWORD PTR arg(4) ;output_height
.loop:
movdqu xmm0, [rsi - 3] ;load src
movdqa xmm1, xmm0
movdqa xmm6, xmm0
movdqa xmm7, xmm0
movdqa xmm2, xmm0
movdqa xmm5, xmm0
movdqa xmm3, xmm0
movdqa xmm4, xmm0
psrldq xmm1, 1
psrldq xmm6, 6
psrldq xmm7, 7
psrldq xmm2, 2
psrldq xmm5, 5
psrldq xmm3, 3
psrldq xmm4, 4
APPLY_FILTER_8 0, 0
movdqu xmm0, [rsi + 5] ;load src
movdqa xmm1, xmm0
movdqa xmm6, xmm0
movdqa xmm7, xmm0
movdqa xmm2, xmm0
movdqa xmm5, xmm0
movdqa xmm3, xmm0
movdqa xmm4, xmm0
psrldq xmm1, 1
psrldq xmm6, 6
psrldq xmm7, 7
psrldq xmm2, 2
psrldq xmm5, 5
psrldq xmm3, 3
psrldq xmm4, 4
APPLY_FILTER_8 0, 8
lea rsi, [rsi + rax]
lea rdi, [rdi + rdx]
dec rcx
jnz .loop
add rsp, 16 * 10
pop rsp
; begin epilog
pop rdi
pop rsi
RESTORE_XMM
UNSHADOW_ARGS
pop rbp
ret