| ; | |
| ; Copyright (c) 2010 The WebM project authors. All Rights Reserved. | |
| ; | |
| ; Use of this source code is governed by a BSD-style license and patent | |
| ; grant that can be found in the LICENSE file in the root of the source | |
| ; tree. All contributing project authors may be found in the AUTHORS | |
| ; file in the root of the source tree. | |
| ; | |
| %include "vpx_ports/x86_abi_support.asm" | |
| %include "asm_enc_offsets.asm" | |
| ; void vp8_regular_quantize_b_sse2 | arg | |
| ; (BLOCK *b, | 0 | |
| ; BLOCKD *d) | 1 | |
| global sym(vp8_regular_quantize_b_sse2) | |
| sym(vp8_regular_quantize_b_sse2): | |
| push rbp | |
| mov rbp, rsp | |
| SAVE_XMM | |
| GET_GOT rbx | |
| push rsi | |
| %if ABI_IS_32BIT | |
| push rdi | |
| %else | |
| %ifidn __OUTPUT_FORMAT__,x64 | |
| push rdi | |
| %endif | |
| %endif | |
| ALIGN_STACK 16, rax | |
| %define BLOCKD_d 0 ; 8 | |
| %define zrun_zbin_boost 8 ; 8 | |
| %define abs_minus_zbin 16 ; 32 | |
| %define temp_qcoeff 48 ; 32 | |
| %define qcoeff 80 ; 32 | |
| %define stack_size 112 | |
| sub rsp, stack_size | |
| ; end prolog | |
| %if ABI_IS_32BIT | |
| mov rdi, arg(0) | |
| %else | |
| %ifidn __OUTPUT_FORMAT__,x64 | |
| mov rdi, rcx ; BLOCK *b | |
| mov [rsp + BLOCKD_d], rdx | |
| %else | |
| ;mov rdi, rdi ; BLOCK *b | |
| mov [rsp + BLOCKD_d], rsi | |
| %endif | |
| %endif | |
| mov rdx, [rdi + vp8_block_coeff] ; coeff_ptr | |
| mov rcx, [rdi + vp8_block_zbin] ; zbin_ptr | |
| movd xmm7, [rdi + vp8_block_zbin_extra] ; zbin_oq_value | |
| ; z | |
| movdqa xmm0, [rdx] | |
| movdqa xmm4, [rdx + 16] | |
| mov rdx, [rdi + vp8_block_round] ; round_ptr | |
| pshuflw xmm7, xmm7, 0 | |
| punpcklwd xmm7, xmm7 ; duplicated zbin_oq_value | |
| movdqa xmm1, xmm0 | |
| movdqa xmm5, xmm4 | |
| ; sz | |
| psraw xmm0, 15 | |
| psraw xmm4, 15 | |
| ; (z ^ sz) | |
| pxor xmm1, xmm0 | |
| pxor xmm5, xmm4 | |
| ; x = abs(z) | |
| psubw xmm1, xmm0 | |
| psubw xmm5, xmm4 | |
| movdqa xmm2, [rcx] | |
| movdqa xmm3, [rcx + 16] | |
| mov rcx, [rdi + vp8_block_quant] ; quant_ptr | |
| ; *zbin_ptr + zbin_oq_value | |
| paddw xmm2, xmm7 | |
| paddw xmm3, xmm7 | |
| ; x - (*zbin_ptr + zbin_oq_value) | |
| psubw xmm1, xmm2 | |
| psubw xmm5, xmm3 | |
| movdqa [rsp + abs_minus_zbin], xmm1 | |
| movdqa [rsp + abs_minus_zbin + 16], xmm5 | |
| ; add (zbin_ptr + zbin_oq_value) back | |
| paddw xmm1, xmm2 | |
| paddw xmm5, xmm3 | |
| movdqa xmm2, [rdx] | |
| movdqa xmm6, [rdx + 16] | |
| movdqa xmm3, [rcx] | |
| movdqa xmm7, [rcx + 16] | |
| ; x + round | |
| paddw xmm1, xmm2 | |
| paddw xmm5, xmm6 | |
| ; y = x * quant_ptr >> 16 | |
| pmulhw xmm3, xmm1 | |
| pmulhw xmm7, xmm5 | |
| ; y += x | |
| paddw xmm1, xmm3 | |
| paddw xmm5, xmm7 | |
| movdqa [rsp + temp_qcoeff], xmm1 | |
| movdqa [rsp + temp_qcoeff + 16], xmm5 | |
| pxor xmm6, xmm6 | |
| ; zero qcoeff | |
| movdqa [rsp + qcoeff], xmm6 | |
| movdqa [rsp + qcoeff + 16], xmm6 | |
| mov rsi, [rdi + vp8_block_zrun_zbin_boost] ; zbin_boost_ptr | |
| mov rax, [rdi + vp8_block_quant_shift] ; quant_shift_ptr | |
| mov [rsp + zrun_zbin_boost], rsi | |
| %macro ZIGZAG_LOOP 1 | |
| movsx edx, WORD PTR[GLOBAL(zig_zag + (%1 * 2))] ; rc | |
| ; x | |
| movsx ecx, WORD PTR[rsp + abs_minus_zbin + rdx *2] | |
| ; if (x >= zbin) | |
| sub cx, WORD PTR[rsi] ; x - zbin | |
| lea rsi, [rsi + 2] ; zbin_boost_ptr++ | |
| jl rq_zigzag_loop_%1 ; x < zbin | |
| movsx edi, WORD PTR[rsp + temp_qcoeff + rdx *2] | |
| ; downshift by quant_shift[rdx] | |
| movsx ecx, WORD PTR[rax + rdx*2] ; quant_shift_ptr[rc] | |
| sar edi, cl ; also sets Z bit | |
| je rq_zigzag_loop_%1 ; !y | |
| mov WORD PTR[rsp + qcoeff + rdx*2], di ;qcoeff_ptr[rc] = temp_qcoeff[rc] | |
| mov rsi, [rsp + zrun_zbin_boost] ; reset to b->zrun_zbin_boost | |
| rq_zigzag_loop_%1: | |
| %endmacro | |
| ZIGZAG_LOOP 0 | |
| ZIGZAG_LOOP 1 | |
| ZIGZAG_LOOP 2 | |
| ZIGZAG_LOOP 3 | |
| ZIGZAG_LOOP 4 | |
| ZIGZAG_LOOP 5 | |
| ZIGZAG_LOOP 6 | |
| ZIGZAG_LOOP 7 | |
| ZIGZAG_LOOP 8 | |
| ZIGZAG_LOOP 9 | |
| ZIGZAG_LOOP 10 | |
| ZIGZAG_LOOP 11 | |
| ZIGZAG_LOOP 12 | |
| ZIGZAG_LOOP 13 | |
| ZIGZAG_LOOP 14 | |
| ZIGZAG_LOOP 15 | |
| movdqa xmm2, [rsp + qcoeff] | |
| movdqa xmm3, [rsp + qcoeff + 16] | |
| %if ABI_IS_32BIT | |
| mov rdi, arg(1) | |
| %else | |
| mov rdi, [rsp + BLOCKD_d] | |
| %endif | |
| mov rcx, [rdi + vp8_blockd_dequant] ; dequant_ptr | |
| mov rsi, [rdi + vp8_blockd_dqcoeff] ; dqcoeff_ptr | |
| ; y ^ sz | |
| pxor xmm2, xmm0 | |
| pxor xmm3, xmm4 | |
| ; x = (y ^ sz) - sz | |
| psubw xmm2, xmm0 | |
| psubw xmm3, xmm4 | |
| ; dequant | |
| movdqa xmm0, [rcx] | |
| movdqa xmm1, [rcx + 16] | |
| mov rcx, [rdi + vp8_blockd_qcoeff] ; qcoeff_ptr | |
| pmullw xmm0, xmm2 | |
| pmullw xmm1, xmm3 | |
| movdqa [rcx], xmm2 ; store qcoeff | |
| movdqa [rcx + 16], xmm3 | |
| movdqa [rsi], xmm0 ; store dqcoeff | |
| movdqa [rsi + 16], xmm1 | |
| ; select the last value (in zig_zag order) for EOB | |
| pcmpeqw xmm2, xmm6 | |
| pcmpeqw xmm3, xmm6 | |
| ; ! | |
| pcmpeqw xmm6, xmm6 | |
| pxor xmm2, xmm6 | |
| pxor xmm3, xmm6 | |
| ; mask inv_zig_zag | |
| pand xmm2, [GLOBAL(inv_zig_zag)] | |
| pand xmm3, [GLOBAL(inv_zig_zag + 16)] | |
| ; select the max value | |
| pmaxsw xmm2, xmm3 | |
| pshufd xmm3, xmm2, 00001110b | |
| pmaxsw xmm2, xmm3 | |
| pshuflw xmm3, xmm2, 00001110b | |
| pmaxsw xmm2, xmm3 | |
| pshuflw xmm3, xmm2, 00000001b | |
| pmaxsw xmm2, xmm3 | |
| movd eax, xmm2 | |
| and eax, 0xff | |
| mov [rdi + vp8_blockd_eob], eax | |
| ; begin epilog | |
| add rsp, stack_size | |
| pop rsp | |
| %if ABI_IS_32BIT | |
| pop rdi | |
| %else | |
| %ifidn __OUTPUT_FORMAT__,x64 | |
| pop rdi | |
| %endif | |
| %endif | |
| pop rsi | |
| RESTORE_GOT | |
| RESTORE_XMM | |
| pop rbp | |
| ret | |
| ; int vp8_fast_quantize_b_impl_sse2 | arg | |
| ; (short *coeff_ptr, | 0 | |
| ; short *qcoeff_ptr, | 1 | |
| ; short *dequant_ptr, | 2 | |
| ; short *inv_scan_order, | 3 | |
| ; short *round_ptr, | 4 | |
| ; short *quant_ptr, | 5 | |
| ; short *dqcoeff_ptr) | 6 | |
| global sym(vp8_fast_quantize_b_impl_sse2) | |
| sym(vp8_fast_quantize_b_impl_sse2): | |
| push rbp | |
| mov rbp, rsp | |
| SHADOW_ARGS_TO_STACK 7 | |
| push rsi | |
| push rdi | |
| ; end prolog | |
| mov rdx, arg(0) ;coeff_ptr | |
| mov rcx, arg(2) ;dequant_ptr | |
| mov rdi, arg(4) ;round_ptr | |
| mov rsi, arg(5) ;quant_ptr | |
| movdqa xmm0, XMMWORD PTR[rdx] | |
| movdqa xmm4, XMMWORD PTR[rdx + 16] | |
| movdqa xmm2, XMMWORD PTR[rdi] ;round lo | |
| movdqa xmm3, XMMWORD PTR[rdi + 16] ;round hi | |
| movdqa xmm1, xmm0 | |
| movdqa xmm5, xmm4 | |
| psraw xmm0, 15 ;sign of z (aka sz) | |
| psraw xmm4, 15 ;sign of z (aka sz) | |
| pxor xmm1, xmm0 | |
| pxor xmm5, xmm4 | |
| psubw xmm1, xmm0 ;x = abs(z) | |
| psubw xmm5, xmm4 ;x = abs(z) | |
| paddw xmm1, xmm2 | |
| paddw xmm5, xmm3 | |
| pmulhw xmm1, XMMWORD PTR[rsi] | |
| pmulhw xmm5, XMMWORD PTR[rsi + 16] | |
| mov rdi, arg(1) ;qcoeff_ptr | |
| mov rsi, arg(6) ;dqcoeff_ptr | |
| movdqa xmm2, XMMWORD PTR[rcx] | |
| movdqa xmm3, XMMWORD PTR[rcx + 16] | |
| pxor xmm1, xmm0 | |
| pxor xmm5, xmm4 | |
| psubw xmm1, xmm0 | |
| psubw xmm5, xmm4 | |
| movdqa XMMWORD PTR[rdi], xmm1 | |
| movdqa XMMWORD PTR[rdi + 16], xmm5 | |
| pmullw xmm2, xmm1 | |
| pmullw xmm3, xmm5 | |
| mov rdi, arg(3) ;inv_scan_order | |
| ; Start with 16 | |
| pxor xmm4, xmm4 ;clear all bits | |
| pcmpeqw xmm1, xmm4 | |
| pcmpeqw xmm5, xmm4 | |
| pcmpeqw xmm4, xmm4 ;set all bits | |
| pxor xmm1, xmm4 | |
| pxor xmm5, xmm4 | |
| pand xmm1, XMMWORD PTR[rdi] | |
| pand xmm5, XMMWORD PTR[rdi+16] | |
| pmaxsw xmm1, xmm5 | |
| ; now down to 8 | |
| pshufd xmm5, xmm1, 00001110b | |
| pmaxsw xmm1, xmm5 | |
| ; only 4 left | |
| pshuflw xmm5, xmm1, 00001110b | |
| pmaxsw xmm1, xmm5 | |
| ; okay, just 2! | |
| pshuflw xmm5, xmm1, 00000001b | |
| pmaxsw xmm1, xmm5 | |
| movd rax, xmm1 | |
| and rax, 0xff | |
| movdqa XMMWORD PTR[rsi], xmm2 ;store dqcoeff | |
| movdqa XMMWORD PTR[rsi + 16], xmm3 ;store dqcoeff | |
| ; begin epilog | |
| pop rdi | |
| pop rsi | |
| UNSHADOW_ARGS | |
| pop rbp | |
| ret | |
| SECTION_RODATA | |
| align 16 | |
| zig_zag: | |
| dw 0x0000, 0x0001, 0x0004, 0x0008 | |
| dw 0x0005, 0x0002, 0x0003, 0x0006 | |
| dw 0x0009, 0x000c, 0x000d, 0x000a | |
| dw 0x0007, 0x000b, 0x000e, 0x000f | |
| inv_zig_zag: | |
| dw 0x0001, 0x0002, 0x0006, 0x0007 | |
| dw 0x0003, 0x0005, 0x0008, 0x000d | |
| dw 0x0004, 0x0009, 0x000c, 0x000e | |
| dw 0x000a, 0x000b, 0x000f, 0x0010 |