| ; |
| ; Copyright (c) 2015 The WebM project authors. All Rights Reserved. |
| ; |
| ; Use of this source code is governed by a BSD-style license |
| ; that can be found in the LICENSE file in the root of the source |
| ; tree. An additional intellectual property rights grant can be found |
| ; in the file PATENTS. All contributing project authors may |
| ; be found in the AUTHORS file in the root of the source tree. |
| ; |
| |
| %include "third_party/x86inc/x86inc.asm" |
| |
| SECTION_RODATA |
| pw_64: times 8 dw 64 |
| |
| ; %define USE_PMULHRSW |
| ; NOTE: pmulhrsw has a latency of 5 cycles. Tests showed a performance loss |
| ; when using this instruction. |
| ; |
| ; The add order below (based on ffvp9) must be followed to prevent outranges. |
| ; x = k0k1 + k4k5 |
| ; y = k2k3 + k6k7 |
| ; z = signed SAT(x + y) |
| |
| SECTION .text |
| %if ARCH_X86_64 |
| %define LOCAL_VARS_SIZE 16*4 |
| %else |
| %define LOCAL_VARS_SIZE 16*6 |
| %endif |
| |
| %macro SETUP_LOCAL_VARS 0 |
| ; TODO(slavarnway): using xmm registers for these on ARCH_X86_64 + |
| ; pmaddubsw has a higher latency on some platforms, this might be eased by |
| ; interleaving the instructions. |
| %define k0k1 [rsp + 16*0] |
| %define k2k3 [rsp + 16*1] |
| %define k4k5 [rsp + 16*2] |
| %define k6k7 [rsp + 16*3] |
| packsswb m4, m4 |
| ; TODO(slavarnway): multiple pshufb instructions had a higher latency on |
| ; some platforms. |
| pshuflw m0, m4, 0b ;k0_k1 |
| pshuflw m1, m4, 01010101b ;k2_k3 |
| pshuflw m2, m4, 10101010b ;k4_k5 |
| pshuflw m3, m4, 11111111b ;k6_k7 |
| punpcklqdq m0, m0 |
| punpcklqdq m1, m1 |
| punpcklqdq m2, m2 |
| punpcklqdq m3, m3 |
| mova k0k1, m0 |
| mova k2k3, m1 |
| mova k4k5, m2 |
| mova k6k7, m3 |
| %if ARCH_X86_64 |
| %define krd m12 |
| %define tmp m13 |
| mova krd, [GLOBAL(pw_64)] |
| %else |
| %define tmp [rsp + 16*4] |
| %define krd [rsp + 16*5] |
| %if CONFIG_PIC=0 |
| mova m6, [GLOBAL(pw_64)] |
| %else |
| ; build constants without accessing global memory |
| pcmpeqb m6, m6 ;all ones |
| psrlw m6, 15 |
| psllw m6, 6 ;aka pw_64 |
| %endif |
| mova krd, m6 |
| %endif |
| %endm |
| |
| %macro HORIZx4_ROW 2 |
| mova %2, %1 |
| punpcklbw %1, %1 |
| punpckhbw %2, %2 |
| |
| mova m3, %2 |
| palignr %2, %1, 1 |
| palignr m3, %1, 5 |
| |
| pmaddubsw %2, k0k1k4k5 |
| pmaddubsw m3, k2k3k6k7 |
| mova m4, %2 ;k0k1 |
| mova m5, m3 ;k2k3 |
| psrldq %2, 8 ;k4k5 |
| psrldq m3, 8 ;k6k7 |
| paddsw %2, m4 |
| paddsw m5, m3 |
| paddsw %2, m5 |
| paddsw %2, krd |
| psraw %2, 7 |
| packuswb %2, %2 |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_HFILTER4 1 |
| cglobal filter_block1d4_%1, 6, 6+(ARCH_X86_64*2), 11, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| packsswb m4, m4 |
| %if ARCH_X86_64 |
| %define k0k1k4k5 m8 |
| %define k2k3k6k7 m9 |
| %define krd m10 |
| %define orig_height r7d |
| mova krd, [GLOBAL(pw_64)] |
| pshuflw k0k1k4k5, m4, 0b ;k0_k1 |
| pshufhw k0k1k4k5, k0k1k4k5, 10101010b ;k0_k1_k4_k5 |
| pshuflw k2k3k6k7, m4, 01010101b ;k2_k3 |
| pshufhw k2k3k6k7, k2k3k6k7, 11111111b ;k2_k3_k6_k7 |
| %else |
| %define k0k1k4k5 [rsp + 16*0] |
| %define k2k3k6k7 [rsp + 16*1] |
| %define krd [rsp + 16*2] |
| %define orig_height [rsp + 16*3] |
| pshuflw m6, m4, 0b ;k0_k1 |
| pshufhw m6, m6, 10101010b ;k0_k1_k4_k5 |
| pshuflw m7, m4, 01010101b ;k2_k3 |
| pshufhw m7, m7, 11111111b ;k2_k3_k6_k7 |
| %if CONFIG_PIC=0 |
| mova m1, [GLOBAL(pw_64)] |
| %else |
| ; build constants without accessing global memory |
| pcmpeqb m1, m1 ;all ones |
| psrlw m1, 15 |
| psllw m1, 6 ;aka pw_64 |
| %endif |
| mova k0k1k4k5, m6 |
| mova k2k3k6k7, m7 |
| mova krd, m1 |
| %endif |
| mov orig_height, heightd |
| shr heightd, 1 |
| .loop: |
| ;Do two rows at once |
| movh m0, [srcq - 3] |
| movh m1, [srcq + 5] |
| punpcklqdq m0, m1 |
| mova m1, m0 |
| movh m2, [srcq + sstrideq - 3] |
| movh m3, [srcq + sstrideq + 5] |
| punpcklqdq m2, m3 |
| mova m3, m2 |
| punpcklbw m0, m0 |
| punpckhbw m1, m1 |
| punpcklbw m2, m2 |
| punpckhbw m3, m3 |
| mova m4, m1 |
| palignr m4, m0, 1 |
| pmaddubsw m4, k0k1k4k5 |
| palignr m1, m0, 5 |
| pmaddubsw m1, k2k3k6k7 |
| mova m7, m3 |
| palignr m7, m2, 1 |
| pmaddubsw m7, k0k1k4k5 |
| palignr m3, m2, 5 |
| pmaddubsw m3, k2k3k6k7 |
| mova m0, m4 ;k0k1 |
| mova m5, m1 ;k2k3 |
| mova m2, m7 ;k0k1 upper |
| psrldq m4, 8 ;k4k5 |
| psrldq m1, 8 ;k6k7 |
| paddsw m4, m0 |
| paddsw m5, m1 |
| mova m1, m3 ;k2k3 upper |
| psrldq m7, 8 ;k4k5 upper |
| psrldq m3, 8 ;k6k7 upper |
| paddsw m7, m2 |
| paddsw m4, m5 |
| paddsw m1, m3 |
| paddsw m7, m1 |
| paddsw m4, krd |
| psraw m4, 7 |
| packuswb m4, m4 |
| paddsw m7, krd |
| psraw m7, 7 |
| packuswb m7, m7 |
| |
| %ifidn %1, h8_avg |
| movd m0, [dstq] |
| pavgb m4, m0 |
| movd m2, [dstq + dstrideq] |
| pavgb m7, m2 |
| %endif |
| movd [dstq], m4 |
| movd [dstq + dstrideq], m7 |
| |
| lea srcq, [srcq + sstrideq ] |
| prefetcht0 [srcq + 4 * sstrideq - 3] |
| lea srcq, [srcq + sstrideq ] |
| lea dstq, [dstq + 2 * dstrideq ] |
| prefetcht0 [srcq + 2 * sstrideq - 3] |
| |
| dec heightd |
| jnz .loop |
| |
| ; Do last row if output_height is odd |
| mov heightd, orig_height |
| and heightd, 1 |
| je .done |
| |
| movh m0, [srcq - 3] ; load src |
| movh m1, [srcq + 5] |
| punpcklqdq m0, m1 |
| |
| HORIZx4_ROW m0, m1 |
| %ifidn %1, h8_avg |
| movd m0, [dstq] |
| pavgb m1, m0 |
| %endif |
| movd [dstq], m1 |
| .done |
| RET |
| %endm |
| |
| %macro HORIZx8_ROW 5 |
| mova %2, %1 |
| punpcklbw %1, %1 |
| punpckhbw %2, %2 |
| |
| mova %3, %2 |
| mova %4, %2 |
| mova %5, %2 |
| |
| palignr %2, %1, 1 |
| palignr %3, %1, 5 |
| palignr %4, %1, 9 |
| palignr %5, %1, 13 |
| |
| pmaddubsw %2, k0k1 |
| pmaddubsw %3, k2k3 |
| pmaddubsw %4, k4k5 |
| pmaddubsw %5, k6k7 |
| paddsw %2, %4 |
| paddsw %5, %3 |
| paddsw %2, %5 |
| paddsw %2, krd |
| psraw %2, 7 |
| packuswb %2, %2 |
| SWAP %1, %2 |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_HFILTER8 1 |
| cglobal filter_block1d8_%1, 6, 6+(ARCH_X86_64*1), 14, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| %if ARCH_X86_64 |
| %define orig_height r7d |
| %else |
| %define orig_height heightmp |
| %endif |
| mov orig_height, heightd |
| shr heightd, 1 |
| |
| .loop: |
| movh m0, [srcq - 3] |
| movh m3, [srcq + 5] |
| movh m4, [srcq + sstrideq - 3] |
| movh m7, [srcq + sstrideq + 5] |
| punpcklqdq m0, m3 |
| mova m1, m0 |
| punpcklbw m0, m0 |
| punpckhbw m1, m1 |
| mova m5, m1 |
| palignr m5, m0, 13 |
| pmaddubsw m5, k6k7 |
| mova m2, m1 |
| mova m3, m1 |
| palignr m1, m0, 1 |
| pmaddubsw m1, k0k1 |
| punpcklqdq m4, m7 |
| mova m6, m4 |
| punpcklbw m4, m4 |
| palignr m2, m0, 5 |
| punpckhbw m6, m6 |
| palignr m3, m0, 9 |
| mova m7, m6 |
| pmaddubsw m2, k2k3 |
| pmaddubsw m3, k4k5 |
| |
| palignr m7, m4, 13 |
| mova m0, m6 |
| palignr m0, m4, 5 |
| pmaddubsw m7, k6k7 |
| paddsw m1, m3 |
| paddsw m2, m5 |
| paddsw m1, m2 |
| mova m5, m6 |
| palignr m6, m4, 1 |
| pmaddubsw m0, k2k3 |
| pmaddubsw m6, k0k1 |
| palignr m5, m4, 9 |
| paddsw m1, krd |
| pmaddubsw m5, k4k5 |
| psraw m1, 7 |
| paddsw m0, m7 |
| %ifidn %1, h8_avg |
| movh m7, [dstq] |
| movh m2, [dstq + dstrideq] |
| %endif |
| packuswb m1, m1 |
| paddsw m6, m5 |
| paddsw m6, m0 |
| paddsw m6, krd |
| psraw m6, 7 |
| packuswb m6, m6 |
| %ifidn %1, h8_avg |
| pavgb m1, m7 |
| pavgb m6, m2 |
| %endif |
| movh [dstq], m1 |
| movh [dstq + dstrideq], m6 |
| |
| lea srcq, [srcq + sstrideq ] |
| prefetcht0 [srcq + 4 * sstrideq - 3] |
| lea srcq, [srcq + sstrideq ] |
| lea dstq, [dstq + 2 * dstrideq ] |
| prefetcht0 [srcq + 2 * sstrideq - 3] |
| dec heightd |
| jnz .loop |
| |
| ;Do last row if output_height is odd |
| mov heightd, orig_height |
| and heightd, 1 |
| je .done |
| |
| movh m0, [srcq - 3] |
| movh m3, [srcq + 5] |
| punpcklqdq m0, m3 |
| |
| HORIZx8_ROW m0, m1, m2, m3, m4 |
| |
| %ifidn %1, h8_avg |
| movh m1, [dstq] |
| pavgb m0, m1 |
| %endif |
| movh [dstq], m0 |
| .done: |
| RET |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_HFILTER16 1 |
| cglobal filter_block1d16_%1, 6, 6+(ARCH_X86_64*0), 14, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| .loop: |
| prefetcht0 [srcq + 2 * sstrideq -3] |
| |
| movh m0, [srcq - 3] |
| movh m4, [srcq + 5] |
| movh m6, [srcq + 13] |
| punpcklqdq m0, m4 |
| mova m7, m0 |
| punpckhbw m0, m0 |
| mova m1, m0 |
| punpcklqdq m4, m6 |
| mova m3, m0 |
| punpcklbw m7, m7 |
| |
| palignr m3, m7, 13 |
| mova m2, m0 |
| pmaddubsw m3, k6k7 |
| palignr m0, m7, 1 |
| pmaddubsw m0, k0k1 |
| palignr m1, m7, 5 |
| pmaddubsw m1, k2k3 |
| palignr m2, m7, 9 |
| pmaddubsw m2, k4k5 |
| paddsw m1, m3 |
| mova m3, m4 |
| punpckhbw m4, m4 |
| mova m5, m4 |
| punpcklbw m3, m3 |
| mova m7, m4 |
| palignr m5, m3, 5 |
| mova m6, m4 |
| palignr m4, m3, 1 |
| pmaddubsw m4, k0k1 |
| pmaddubsw m5, k2k3 |
| palignr m6, m3, 9 |
| pmaddubsw m6, k4k5 |
| palignr m7, m3, 13 |
| pmaddubsw m7, k6k7 |
| paddsw m0, m2 |
| paddsw m0, m1 |
| %ifidn %1, h8_avg |
| mova m1, [dstq] |
| %endif |
| paddsw m4, m6 |
| paddsw m5, m7 |
| paddsw m4, m5 |
| paddsw m0, krd |
| paddsw m4, krd |
| psraw m0, 7 |
| psraw m4, 7 |
| packuswb m0, m4 |
| %ifidn %1, h8_avg |
| pavgb m0, m1 |
| %endif |
| lea srcq, [srcq + sstrideq] |
| mova [dstq], m0 |
| lea dstq, [dstq + dstrideq] |
| dec heightd |
| jnz .loop |
| RET |
| %endm |
| |
| INIT_XMM ssse3 |
| SUBPIX_HFILTER16 h8 |
| SUBPIX_HFILTER16 h8_avg |
| SUBPIX_HFILTER8 h8 |
| SUBPIX_HFILTER8 h8_avg |
| SUBPIX_HFILTER4 h8 |
| SUBPIX_HFILTER4 h8_avg |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_VFILTER 2 |
| cglobal filter_block1d%2_%1, 6, 6+(ARCH_X86_64*3), 14, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| %if ARCH_X86_64 |
| %define src1q r7 |
| %define sstride6q r8 |
| %define dst_stride dstrideq |
| %else |
| %define src1q filterq |
| %define sstride6q dstrideq |
| %define dst_stride dstridemp |
| %endif |
| mov src1q, srcq |
| add src1q, sstrideq |
| lea sstride6q, [sstrideq + sstrideq * 4] |
| add sstride6q, sstrideq ;pitch * 6 |
| |
| %ifidn %2, 8 |
| %define movx movh |
| %else |
| %define movx movd |
| %endif |
| .loop: |
| movx m0, [srcq ] ;A |
| movx m1, [srcq + sstrideq ] ;B |
| punpcklbw m0, m1 ;A B |
| movx m2, [srcq + sstrideq * 2 ] ;C |
| pmaddubsw m0, k0k1 |
| mova m6, m2 |
| movx m3, [src1q + sstrideq * 2] ;D |
| punpcklbw m2, m3 ;C D |
| pmaddubsw m2, k2k3 |
| movx m4, [srcq + sstrideq * 4 ] ;E |
| mova m7, m4 |
| movx m5, [src1q + sstrideq * 4] ;F |
| punpcklbw m4, m5 ;E F |
| pmaddubsw m4, k4k5 |
| punpcklbw m1, m6 ;A B next iter |
| movx m6, [srcq + sstride6q ] ;G |
| punpcklbw m5, m6 ;E F next iter |
| punpcklbw m3, m7 ;C D next iter |
| pmaddubsw m5, k4k5 |
| movx m7, [src1q + sstride6q ] ;H |
| punpcklbw m6, m7 ;G H |
| pmaddubsw m6, k6k7 |
| pmaddubsw m3, k2k3 |
| pmaddubsw m1, k0k1 |
| paddsw m0, m4 |
| paddsw m2, m6 |
| movx m6, [srcq + sstrideq * 8 ] ;H next iter |
| punpcklbw m7, m6 |
| pmaddubsw m7, k6k7 |
| paddsw m0, m2 |
| paddsw m0, krd |
| psraw m0, 7 |
| paddsw m1, m5 |
| packuswb m0, m0 |
| |
| paddsw m3, m7 |
| paddsw m1, m3 |
| paddsw m1, krd |
| psraw m1, 7 |
| lea srcq, [srcq + sstrideq * 2 ] |
| lea src1q, [src1q + sstrideq * 2] |
| packuswb m1, m1 |
| |
| %ifidn %1, v8_avg |
| movx m2, [dstq] |
| pavgb m0, m2 |
| %endif |
| movx [dstq], m0 |
| add dstq, dst_stride |
| %ifidn %1, v8_avg |
| movx m3, [dstq] |
| pavgb m1, m3 |
| %endif |
| movx [dstq], m1 |
| add dstq, dst_stride |
| sub heightd, 2 |
| cmp heightd, 1 |
| jg .loop |
| |
| cmp heightd, 0 |
| je .done |
| |
| movx m0, [srcq ] ;A |
| movx m1, [srcq + sstrideq ] ;B |
| movx m6, [srcq + sstride6q ] ;G |
| punpcklbw m0, m1 ;A B |
| movx m7, [src1q + sstride6q ] ;H |
| pmaddubsw m0, k0k1 |
| movx m2, [srcq + sstrideq * 2 ] ;C |
| punpcklbw m6, m7 ;G H |
| movx m3, [src1q + sstrideq * 2] ;D |
| pmaddubsw m6, k6k7 |
| movx m4, [srcq + sstrideq * 4 ] ;E |
| punpcklbw m2, m3 ;C D |
| movx m5, [src1q + sstrideq * 4] ;F |
| punpcklbw m4, m5 ;E F |
| pmaddubsw m2, k2k3 |
| pmaddubsw m4, k4k5 |
| paddsw m2, m6 |
| paddsw m0, m4 |
| paddsw m0, m2 |
| paddsw m0, krd |
| psraw m0, 7 |
| packuswb m0, m0 |
| %ifidn %1, v8_avg |
| movx m1, [dstq] |
| pavgb m0, m1 |
| %endif |
| movx [dstq], m0 |
| .done: |
| RET |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_VFILTER16 1 |
| cglobal filter_block1d16_%1, 6, 6+(ARCH_X86_64*3), 14, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| %if ARCH_X86_64 |
| %define src1q r7 |
| %define sstride6q r8 |
| %define dst_stride dstrideq |
| %else |
| %define src1q filterq |
| %define sstride6q dstrideq |
| %define dst_stride dstridemp |
| %endif |
| mov src1q, srcq |
| add src1q, sstrideq |
| lea sstride6q, [sstrideq + sstrideq * 4] |
| add sstride6q, sstrideq ;pitch * 6 |
| |
| .loop: |
| movh m0, [srcq ] ;A |
| movh m1, [srcq + sstrideq ] ;B |
| movh m2, [srcq + sstrideq * 2 ] ;C |
| movh m3, [src1q + sstrideq * 2] ;D |
| movh m4, [srcq + sstrideq * 4 ] ;E |
| movh m5, [src1q + sstrideq * 4] ;F |
| |
| punpcklbw m0, m1 ;A B |
| movh m6, [srcq + sstride6q] ;G |
| punpcklbw m2, m3 ;C D |
| movh m7, [src1q + sstride6q] ;H |
| punpcklbw m4, m5 ;E F |
| pmaddubsw m0, k0k1 |
| movh m3, [srcq + 8] ;A |
| pmaddubsw m2, k2k3 |
| punpcklbw m6, m7 ;G H |
| movh m5, [srcq + sstrideq + 8] ;B |
| pmaddubsw m4, k4k5 |
| punpcklbw m3, m5 ;A B |
| movh m7, [srcq + sstrideq * 2 + 8] ;C |
| pmaddubsw m6, k6k7 |
| movh m5, [src1q + sstrideq * 2 + 8] ;D |
| punpcklbw m7, m5 ;C D |
| paddsw m2, m6 |
| pmaddubsw m3, k0k1 |
| movh m1, [srcq + sstrideq * 4 + 8] ;E |
| paddsw m0, m4 |
| pmaddubsw m7, k2k3 |
| movh m6, [src1q + sstrideq * 4 + 8] ;F |
| punpcklbw m1, m6 ;E F |
| paddsw m0, m2 |
| paddsw m0, krd |
| movh m2, [srcq + sstride6q + 8] ;G |
| pmaddubsw m1, k4k5 |
| movh m5, [src1q + sstride6q + 8] ;H |
| psraw m0, 7 |
| punpcklbw m2, m5 ;G H |
| pmaddubsw m2, k6k7 |
| %ifidn %1, v8_avg |
| mova m4, [dstq] |
| %endif |
| movh [dstq], m0 |
| paddsw m7, m2 |
| paddsw m3, m1 |
| paddsw m3, m7 |
| paddsw m3, krd |
| psraw m3, 7 |
| packuswb m0, m3 |
| |
| add srcq, sstrideq |
| add src1q, sstrideq |
| %ifidn %1, v8_avg |
| pavgb m0, m4 |
| %endif |
| mova [dstq], m0 |
| add dstq, dst_stride |
| dec heightd |
| jnz .loop |
| RET |
| %endm |
| |
| INIT_XMM ssse3 |
| SUBPIX_VFILTER16 v8 |
| SUBPIX_VFILTER16 v8_avg |
| SUBPIX_VFILTER v8, 8 |
| SUBPIX_VFILTER v8_avg, 8 |
| SUBPIX_VFILTER v8, 4 |
| SUBPIX_VFILTER v8_avg, 4 |
