| ; |
| ; Copyright (c) 2016, Alliance for Open Media. All rights reserved |
| ; |
| ; This source code is subject to the terms of the BSD 2 Clause License and |
| ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| ; was not distributed with this source code in the LICENSE file, you can |
| ; obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| ; Media Patent License 1.0 was not distributed with this source code in the |
| ; PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| ; |
| |
| ; |
| |
| %include "third_party/x86inc/x86inc.asm" |
| |
| SECTION_RODATA |
| pw_64: times 8 dw 64 |
| even_byte_mask: times 8 dw 0x00ff |
| |
| ; %define USE_PMULHRSW |
| ; NOTE: pmulhrsw has a latency of 5 cycles. Tests showed a performance loss |
| ; when using this instruction. |
| ; |
| ; The add order below (based on ffav1) must be followed to prevent outranges. |
| ; x = k0k1 + k4k5 |
| ; y = k2k3 + k6k7 |
| ; z = signed SAT(x + y) |
| |
| SECTION .text |
| %define LOCAL_VARS_SIZE 16*6 |
| |
| %macro SETUP_LOCAL_VARS 0 |
| ; TODO(slavarnway): using xmm registers for these on ARCH_X86_64 + |
| ; pmaddubsw has a higher latency on some platforms, this might be eased by |
| ; interleaving the instructions. |
| %define k0k1 [rsp + 16*0] |
| %define k2k3 [rsp + 16*1] |
| %define k4k5 [rsp + 16*2] |
| %define k6k7 [rsp + 16*3] |
| packsswb m4, m4 |
| ; TODO(slavarnway): multiple pshufb instructions had a higher latency on |
| ; some platforms. |
| pshuflw m0, m4, 0b ;k0_k1 |
| pshuflw m1, m4, 01010101b ;k2_k3 |
| pshuflw m2, m4, 10101010b ;k4_k5 |
| pshuflw m3, m4, 11111111b ;k6_k7 |
| punpcklqdq m0, m0 |
| punpcklqdq m1, m1 |
| punpcklqdq m2, m2 |
| punpcklqdq m3, m3 |
| mova k0k1, m0 |
| mova k2k3, m1 |
| mova k4k5, m2 |
| mova k6k7, m3 |
| %if ARCH_X86_64 |
| %define krd m12 |
| %define tmp0 [rsp + 16*4] |
| %define tmp1 [rsp + 16*5] |
| mova krd, [GLOBAL(pw_64)] |
| %else |
| %define krd [rsp + 16*4] |
| %if CONFIG_PIC=0 |
| mova m6, [GLOBAL(pw_64)] |
| %else |
| ; build constants without accessing global memory |
| pcmpeqb m6, m6 ;all ones |
| psrlw m6, 15 |
| psllw m6, 6 ;aka pw_64 |
| %endif |
| mova krd, m6 |
| %endif |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %if ARCH_X86_64 |
| %define LOCAL_VARS_SIZE_H4 0 |
| %else |
| %define LOCAL_VARS_SIZE_H4 16*4 |
| %endif |
| |
| %macro SUBPIX_HFILTER4 1 |
| cglobal filter_block1d4_%1, 6, 6, 11, LOCAL_VARS_SIZE_H4, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| packsswb m4, m4 |
| %if ARCH_X86_64 |
| %define k0k1k4k5 m8 |
| %define k2k3k6k7 m9 |
| %define krd m10 |
| mova krd, [GLOBAL(pw_64)] |
| pshuflw k0k1k4k5, m4, 0b ;k0_k1 |
| pshufhw k0k1k4k5, k0k1k4k5, 10101010b ;k0_k1_k4_k5 |
| pshuflw k2k3k6k7, m4, 01010101b ;k2_k3 |
| pshufhw k2k3k6k7, k2k3k6k7, 11111111b ;k2_k3_k6_k7 |
| %else |
| %define k0k1k4k5 [rsp + 16*0] |
| %define k2k3k6k7 [rsp + 16*1] |
| %define krd [rsp + 16*2] |
| pshuflw m6, m4, 0b ;k0_k1 |
| pshufhw m6, m6, 10101010b ;k0_k1_k4_k5 |
| pshuflw m7, m4, 01010101b ;k2_k3 |
| pshufhw m7, m7, 11111111b ;k2_k3_k6_k7 |
| %if CONFIG_PIC=0 |
| mova m1, [GLOBAL(pw_64)] |
| %else |
| ; build constants without accessing global memory |
| pcmpeqb m1, m1 ;all ones |
| psrlw m1, 15 |
| psllw m1, 6 ;aka pw_64 |
| %endif |
| mova k0k1k4k5, m6 |
| mova k2k3k6k7, m7 |
| mova krd, m1 |
| %endif |
| dec heightd |
| |
| .loop: |
| ;Do two rows at once |
| movu m4, [srcq - 3] |
| movu m5, [srcq + sstrideq - 3] |
| punpckhbw m1, m4, m4 |
| punpcklbw m4, m4 |
| punpckhbw m3, m5, m5 |
| punpcklbw m5, m5 |
| palignr m0, m1, m4, 1 |
| pmaddubsw m0, k0k1k4k5 |
| palignr m1, m4, 5 |
| pmaddubsw m1, k2k3k6k7 |
| palignr m2, m3, m5, 1 |
| pmaddubsw m2, k0k1k4k5 |
| palignr m3, m5, 5 |
| pmaddubsw m3, k2k3k6k7 |
| punpckhqdq m4, m0, m2 |
| punpcklqdq m0, m2 |
| punpckhqdq m5, m1, m3 |
| punpcklqdq m1, m3 |
| paddsw m0, m4 |
| paddsw m1, m5 |
| %ifidn %1, h8_avg |
| movd m4, [dstq] |
| movd m5, [dstq + dstrideq] |
| %endif |
| paddsw m0, m1 |
| paddsw m0, krd |
| psraw m0, 7 |
| %ifidn %1, h8_add_src |
| pxor m3, m3 |
| movu m4, [srcq] |
| movu m5, [srcq + sstrideq] |
| punpckldq m4, m5 ; Bytes 0,1,2,3 from row 0, then 0,1,2,3 from row 2 |
| punpcklbw m4, m3 |
| paddsw m0, m4 |
| %endif |
| packuswb m0, m0 |
| psrldq m1, m0, 4 |
| |
| %ifidn %1, h8_avg |
| pavgb m0, m4 |
| pavgb m1, m5 |
| %endif |
| movd [dstq], m0 |
| movd [dstq + dstrideq], m1 |
| |
| lea srcq, [srcq + sstrideq ] |
| prefetcht0 [srcq + 4 * sstrideq - 3] |
| lea srcq, [srcq + sstrideq ] |
| lea dstq, [dstq + 2 * dstrideq ] |
| prefetcht0 [srcq + 2 * sstrideq - 3] |
| |
| sub heightd, 2 |
| jg .loop |
| |
| ; Do last row if output_height is odd |
| jne .done |
| |
| movu m4, [srcq - 3] |
| punpckhbw m1, m4, m4 |
| punpcklbw m4, m4 |
| palignr m0, m1, m4, 1 |
| palignr m1, m4, 5 |
| pmaddubsw m0, k0k1k4k5 |
| pmaddubsw m1, k2k3k6k7 |
| psrldq m2, m0, 8 |
| psrldq m3, m1, 8 |
| paddsw m0, m2 |
| paddsw m1, m3 |
| paddsw m0, m1 |
| paddsw m0, krd |
| psraw m0, 7 |
| %ifidn %1, h8_add_src |
| pxor m3, m3 |
| movu m4, [srcq] |
| punpcklbw m4, m3 |
| paddsw m0, m4 |
| %endif |
| packuswb m0, m0 |
| %ifidn %1, h8_avg |
| movd m4, [dstq] |
| pavgb m0, m4 |
| %endif |
| movd [dstq], m0 |
| .done: |
| REP_RET |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_HFILTER8 1 |
| cglobal filter_block1d8_%1, 6, 6, 14, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| dec heightd |
| |
| .loop: |
| ;Do two rows at once |
| movu m0, [srcq - 3] |
| movu m4, [srcq + sstrideq - 3] |
| punpckhbw m1, m0, m0 |
| punpcklbw m0, m0 |
| palignr m5, m1, m0, 13 |
| pmaddubsw m5, k6k7 |
| palignr m2, m1, m0, 5 |
| palignr m3, m1, m0, 9 |
| palignr m1, m0, 1 |
| pmaddubsw m1, k0k1 |
| punpckhbw m6, m4, m4 |
| punpcklbw m4, m4 |
| pmaddubsw m2, k2k3 |
| pmaddubsw m3, k4k5 |
| |
| palignr m7, m6, m4, 13 |
| palignr m0, m6, m4, 5 |
| pmaddubsw m7, k6k7 |
| paddsw m1, m3 |
| paddsw m2, m5 |
| paddsw m1, m2 |
| %ifidn %1, h8_avg |
| movh m2, [dstq] |
| movhps m2, [dstq + dstrideq] |
| %endif |
| palignr m5, m6, m4, 9 |
| palignr m6, m4, 1 |
| pmaddubsw m0, k2k3 |
| pmaddubsw m6, k0k1 |
| paddsw m1, krd |
| pmaddubsw m5, k4k5 |
| psraw m1, 7 |
| paddsw m0, m7 |
| paddsw m6, m5 |
| paddsw m6, m0 |
| paddsw m6, krd |
| psraw m6, 7 |
| %ifidn %1, h8_add_src |
| pxor m3, m3 |
| movu m4, [srcq] |
| movu m5, [srcq + sstrideq] |
| punpcklbw m4, m3 |
| punpcklbw m5, m3 |
| paddsw m1, m4 |
| paddsw m6, m5 |
| %endif |
| packuswb m1, m6 |
| %ifidn %1, h8_avg |
| pavgb m1, m2 |
| %endif |
| movh [dstq], m1 |
| movhps [dstq + dstrideq], m1 |
| |
| lea srcq, [srcq + sstrideq ] |
| prefetcht0 [srcq + 4 * sstrideq - 3] |
| lea srcq, [srcq + sstrideq ] |
| lea dstq, [dstq + 2 * dstrideq ] |
| prefetcht0 [srcq + 2 * sstrideq - 3] |
| sub heightd, 2 |
| jg .loop |
| |
| ; Do last row if output_height is odd |
| jne .done |
| |
| movu m0, [srcq - 3] |
| punpckhbw m3, m0, m0 |
| punpcklbw m0, m0 |
| palignr m1, m3, m0, 1 |
| palignr m2, m3, m0, 5 |
| palignr m4, m3, m0, 13 |
| palignr m3, m0, 9 |
| pmaddubsw m1, k0k1 |
| pmaddubsw m2, k2k3 |
| pmaddubsw m3, k4k5 |
| pmaddubsw m4, k6k7 |
| paddsw m1, m3 |
| paddsw m4, m2 |
| paddsw m1, m4 |
| paddsw m1, krd |
| psraw m1, 7 |
| %ifidn %1, h8_add_src |
| pxor m6, m6 |
| movu m5, [srcq] |
| punpcklbw m5, m6 |
| paddsw m1, m5 |
| %endif |
| packuswb m1, m1 |
| %ifidn %1, h8_avg |
| movh m0, [dstq] |
| pavgb m1, m0 |
| %endif |
| movh [dstq], m1 |
| .done: |
| REP_RET |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_HFILTER16 1 |
| cglobal filter_block1d16_%1, 6, 6, 14, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| |
| .loop: |
| prefetcht0 [srcq + 2 * sstrideq -3] |
| |
| movu m0, [srcq - 3] |
| movu m4, [srcq - 2] |
| pmaddubsw m0, k0k1 |
| pmaddubsw m4, k0k1 |
| movu m1, [srcq - 1] |
| movu m5, [srcq + 0] |
| pmaddubsw m1, k2k3 |
| pmaddubsw m5, k2k3 |
| movu m2, [srcq + 1] |
| movu m6, [srcq + 2] |
| pmaddubsw m2, k4k5 |
| pmaddubsw m6, k4k5 |
| movu m3, [srcq + 3] |
| movu m7, [srcq + 4] |
| pmaddubsw m3, k6k7 |
| pmaddubsw m7, k6k7 |
| paddsw m0, m2 |
| paddsw m1, m3 |
| paddsw m0, m1 |
| paddsw m4, m6 |
| paddsw m5, m7 |
| paddsw m4, m5 |
| paddsw m0, krd |
| paddsw m4, krd |
| psraw m0, 7 |
| psraw m4, 7 |
| %ifidn %1, h8_add_src |
| movu m5, [srcq] |
| mova m7, m5 |
| pand m5, [even_byte_mask] |
| psrlw m7, 8 |
| paddsw m0, m5 |
| paddsw m4, m7 |
| %endif |
| packuswb m0, m0 |
| packuswb m4, m4 |
| punpcklbw m0, m4 |
| %ifidn %1, h8_avg |
| pavgb m0, [dstq] |
| %endif |
| lea srcq, [srcq + sstrideq] |
| mova [dstq], m0 |
| lea dstq, [dstq + dstrideq] |
| dec heightd |
| jnz .loop |
| REP_RET |
| %endm |
| |
| INIT_XMM ssse3 |
| SUBPIX_HFILTER16 h8 |
| SUBPIX_HFILTER16 h8_avg |
| SUBPIX_HFILTER8 h8 |
| SUBPIX_HFILTER8 h8_avg |
| SUBPIX_HFILTER4 h8 |
| SUBPIX_HFILTER4 h8_avg |
| |
| %if CONFIG_LOOP_RESTORATION |
| SUBPIX_HFILTER16 h8_add_src |
| SUBPIX_HFILTER8 h8_add_src |
| SUBPIX_HFILTER4 h8_add_src |
| %endif |
| |
| ;------------------------------------------------------------------------------- |
| |
| ; TODO(Linfeng): Detect cpu type and choose the code with better performance. |
| %define X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON 1 |
| |
| %if ARCH_X86_64 && X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON |
| %define NUM_GENERAL_REG_USED 9 |
| %else |
| %define NUM_GENERAL_REG_USED 6 |
| %endif |
| |
| %macro SUBPIX_VFILTER 2 |
| cglobal filter_block1d%2_%1, 6, NUM_GENERAL_REG_USED, 15, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| |
| %ifidn %2, 8 |
| %define movx movh |
| %else |
| %define movx movd |
| %endif |
| |
| dec heightd |
| |
| %if ARCH_X86 || X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON |
| |
| %if ARCH_X86_64 |
| %define src1q r7 |
| %define sstride6q r8 |
| %define dst_stride dstrideq |
| %else |
| %define src1q filterq |
| %define sstride6q dstrideq |
| %define dst_stride dstridemp |
| %endif |
| mov src1q, srcq |
| add src1q, sstrideq |
| lea sstride6q, [sstrideq + sstrideq * 4] |
| add sstride6q, sstrideq ;pitch * 6 |
| |
| .loop: |
| ;Do two rows at once |
| movx m0, [srcq ] ;A |
| movx m1, [src1q ] ;B |
| punpcklbw m0, m1 ;A B |
| movx m2, [srcq + sstrideq * 2 ] ;C |
| pmaddubsw m0, k0k1 |
| mova m6, m2 |
| movx m3, [src1q + sstrideq * 2] ;D |
| punpcklbw m2, m3 ;C D |
| pmaddubsw m2, k2k3 |
| movx m4, [srcq + sstrideq * 4 ] ;E |
| mova m7, m4 |
| movx m5, [src1q + sstrideq * 4] ;F |
| punpcklbw m4, m5 ;E F |
| pmaddubsw m4, k4k5 |
| punpcklbw m1, m6 ;A B next iter |
| movx m6, [srcq + sstride6q ] ;G |
| punpcklbw m5, m6 ;E F next iter |
| punpcklbw m3, m7 ;C D next iter |
| pmaddubsw m5, k4k5 |
| movx m7, [src1q + sstride6q ] ;H |
| punpcklbw m6, m7 ;G H |
| pmaddubsw m6, k6k7 |
| pmaddubsw m3, k2k3 |
| pmaddubsw m1, k0k1 |
| paddsw m0, m4 |
| paddsw m2, m6 |
| movx m6, [srcq + sstrideq * 8 ] ;H next iter |
| punpcklbw m7, m6 |
| pmaddubsw m7, k6k7 |
| paddsw m0, m2 |
| paddsw m0, krd |
| psraw m0, 7 |
| paddsw m1, m5 |
| %ifidn %1, v8_add_src |
| pxor m6, m6 |
| movu m4, [srcq] |
| punpcklbw m4, m6 |
| paddsw m0, m4 |
| %endif |
| packuswb m0, m0 |
| |
| paddsw m3, m7 |
| paddsw m1, m3 |
| paddsw m1, krd |
| psraw m1, 7 |
| %ifidn %1, v8_add_src |
| movu m4, [src1q] |
| punpcklbw m4, m6 |
| paddsw m1, m4 |
| %endif |
| lea srcq, [srcq + sstrideq * 2 ] |
| lea src1q, [src1q + sstrideq * 2] |
| packuswb m1, m1 |
| |
| %ifidn %1, v8_avg |
| movx m2, [dstq] |
| pavgb m0, m2 |
| %endif |
| movx [dstq], m0 |
| add dstq, dst_stride |
| %ifidn %1, v8_avg |
| movx m3, [dstq] |
| pavgb m1, m3 |
| %endif |
| movx [dstq], m1 |
| add dstq, dst_stride |
| sub heightd, 2 |
| jg .loop |
| |
| ; Do last row if output_height is odd |
| jne .done |
| |
| movx m0, [srcq ] ;A |
| movx m1, [srcq + sstrideq ] ;B |
| movx m6, [srcq + sstride6q ] ;G |
| punpcklbw m0, m1 ;A B |
| movx m7, [src1q + sstride6q ] ;H |
| pmaddubsw m0, k0k1 |
| movx m2, [srcq + sstrideq * 2 ] ;C |
| punpcklbw m6, m7 ;G H |
| movx m3, [src1q + sstrideq * 2] ;D |
| pmaddubsw m6, k6k7 |
| movx m4, [srcq + sstrideq * 4 ] ;E |
| punpcklbw m2, m3 ;C D |
| movx m5, [src1q + sstrideq * 4] ;F |
| punpcklbw m4, m5 ;E F |
| pmaddubsw m2, k2k3 |
| pmaddubsw m4, k4k5 |
| paddsw m2, m6 |
| paddsw m0, m4 |
| paddsw m0, m2 |
| paddsw m0, krd |
| psraw m0, 7 |
| %ifidn %1, v8_add_src |
| pxor m6, m6 |
| movu m4, [srcq] |
| punpcklbw m4, m6 |
| paddsw m0, m4 |
| %endif |
| packuswb m0, m0 |
| %ifidn %1, v8_avg |
| movx m1, [dstq] |
| pavgb m0, m1 |
| %endif |
| movx [dstq], m0 |
| |
| %else |
| ; ARCH_X86_64 |
| |
| movx m0, [srcq ] ;A |
| movx m1, [srcq + sstrideq ] ;B |
| lea srcq, [srcq + sstrideq * 2 ] |
| movx m2, [srcq] ;C |
| movx m3, [srcq + sstrideq] ;D |
| lea srcq, [srcq + sstrideq * 2 ] |
| movx m4, [srcq] ;E |
| movx m5, [srcq + sstrideq] ;F |
| lea srcq, [srcq + sstrideq * 2 ] |
| movx m6, [srcq] ;G |
| punpcklbw m0, m1 ;A B |
| punpcklbw m1, m2 ;A B next iter |
| punpcklbw m2, m3 ;C D |
| punpcklbw m3, m4 ;C D next iter |
| punpcklbw m4, m5 ;E F |
| punpcklbw m5, m6 ;E F next iter |
| |
| .loop: |
| ;Do two rows at once |
| movx m7, [srcq + sstrideq] ;H |
| lea srcq, [srcq + sstrideq * 2 ] |
| movx m14, [srcq] ;H next iter |
| punpcklbw m6, m7 ;G H |
| punpcklbw m7, m14 ;G H next iter |
| pmaddubsw m8, m0, k0k1 |
| pmaddubsw m9, m1, k0k1 |
| mova m0, m2 |
| mova m1, m3 |
| pmaddubsw m10, m2, k2k3 |
| pmaddubsw m11, m3, k2k3 |
| mova m2, m4 |
| mova m3, m5 |
| pmaddubsw m4, k4k5 |
| pmaddubsw m5, k4k5 |
| paddsw m8, m4 |
| paddsw m9, m5 |
| mova m4, m6 |
| mova m5, m7 |
| pmaddubsw m6, k6k7 |
| pmaddubsw m7, k6k7 |
| paddsw m10, m6 |
| paddsw m11, m7 |
| paddsw m8, m10 |
| paddsw m9, m11 |
| mova m6, m14 |
| paddsw m8, krd |
| paddsw m9, krd |
| psraw m8, 7 |
| psraw m9, 7 |
| %ifidn %2, 4 |
| packuswb m8, m8 |
| packuswb m9, m9 |
| %else |
| packuswb m8, m9 |
| %endif |
| |
| %ifidn %1, v8_avg |
| movx m7, [dstq] |
| %ifidn %2, 4 |
| movx m10, [dstq + dstrideq] |
| pavgb m9, m10 |
| %else |
| movhpd m7, [dstq + dstrideq] |
| %endif |
| pavgb m8, m7 |
| %endif |
| movx [dstq], m8 |
| %ifidn %2, 4 |
| movx [dstq + dstrideq], m9 |
| %else |
| movhpd [dstq + dstrideq], m8 |
| %endif |
| |
| lea dstq, [dstq + dstrideq * 2 ] |
| sub heightd, 2 |
| jg .loop |
| |
| ; Do last row if output_height is odd |
| jne .done |
| |
| movx m7, [srcq + sstrideq] ;H |
| punpcklbw m6, m7 ;G H |
| pmaddubsw m0, k0k1 |
| pmaddubsw m2, k2k3 |
| pmaddubsw m4, k4k5 |
| pmaddubsw m6, k6k7 |
| paddsw m0, m4 |
| paddsw m2, m6 |
| paddsw m0, m2 |
| paddsw m0, krd |
| psraw m0, 7 |
| packuswb m0, m0 |
| %ifidn %1, v8_avg |
| movx m1, [dstq] |
| pavgb m0, m1 |
| %endif |
| movx [dstq], m0 |
| |
| %endif ; ARCH_X86_64 |
| |
| .done: |
| REP_RET |
| |
| %endm |
| |
| ;------------------------------------------------------------------------------- |
| %macro SUBPIX_VFILTER16 1 |
| cglobal filter_block1d16_%1, 6, NUM_GENERAL_REG_USED, 16, LOCAL_VARS_SIZE, \ |
| src, sstride, dst, dstride, height, filter |
| mova m4, [filterq] |
| SETUP_LOCAL_VARS |
| |
| %if ARCH_X86 || X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON |
| |
| %if ARCH_X86_64 |
| %define src1q r7 |
| %define sstride6q r8 |
| %define dst_stride dstrideq |
| %else |
| %define src1q filterq |
| %define sstride6q dstrideq |
| %define dst_stride dstridemp |
| %endif |
| lea src1q, [srcq + sstrideq] |
| lea sstride6q, [sstrideq + sstrideq * 4] |
| add sstride6q, sstrideq ;pitch * 6 |
| |
| .loop: |
| movh m0, [srcq ] ;A |
| movh m1, [src1q ] ;B |
| movh m2, [srcq + sstrideq * 2 ] ;C |
| movh m3, [src1q + sstrideq * 2] ;D |
| movh m4, [srcq + sstrideq * 4 ] ;E |
| movh m5, [src1q + sstrideq * 4] ;F |
| |
| punpcklbw m0, m1 ;A B |
| movh m6, [srcq + sstride6q] ;G |
| punpcklbw m2, m3 ;C D |
| movh m7, [src1q + sstride6q] ;H |
| punpcklbw m4, m5 ;E F |
| pmaddubsw m0, k0k1 |
| movh m3, [srcq + 8] ;A |
| pmaddubsw m2, k2k3 |
| punpcklbw m6, m7 ;G H |
| movh m5, [srcq + sstrideq + 8] ;B |
| pmaddubsw m4, k4k5 |
| punpcklbw m3, m5 ;A B |
| movh m7, [srcq + sstrideq * 2 + 8] ;C |
| pmaddubsw m6, k6k7 |
| movh m5, [src1q + sstrideq * 2 + 8] ;D |
| punpcklbw m7, m5 ;C D |
| paddsw m2, m6 |
| pmaddubsw m3, k0k1 |
| movh m1, [srcq + sstrideq * 4 + 8] ;E |
| paddsw m0, m4 |
| pmaddubsw m7, k2k3 |
| movh m6, [src1q + sstrideq * 4 + 8] ;F |
| punpcklbw m1, m6 ;E F |
| paddsw m0, m2 |
| paddsw m0, krd |
| movh m2, [srcq + sstride6q + 8] ;G |
| pmaddubsw m1, k4k5 |
| movh m5, [src1q + sstride6q + 8] ;H |
| psraw m0, 7 |
| punpcklbw m2, m5 ;G H |
| pmaddubsw m2, k6k7 |
| paddsw m7, m2 |
| paddsw m3, m1 |
| paddsw m3, m7 |
| paddsw m3, krd |
| psraw m3, 7 |
| %ifidn %1, v8_add_src |
| pxor m6, m6 |
| movu m4, [src1q + 2 * sstrideq] ; Fetch from 3 rows down |
| mova m5, m4 |
| punpcklbw m4, m6 |
| punpckhbw m5, m6 |
| paddsw m0, m4 |
| paddsw m3, m5 |
| %endif |
| packuswb m0, m3 |
| |
| add srcq, sstrideq |
| add src1q, sstrideq |
| %ifidn %1, v8_avg |
| pavgb m0, [dstq] |
| %endif |
| mova [dstq], m0 |
| add dstq, dst_stride |
| dec heightd |
| jnz .loop |
| REP_RET |
| |
| %else |
| ; ARCH_X86_64 |
| dec heightd |
| |
| movu m1, [srcq ] ;A |
| movu m3, [srcq + sstrideq ] ;B |
| lea srcq, [srcq + sstrideq * 2] |
| punpcklbw m0, m1, m3 ;A B |
| punpckhbw m1, m3 ;A B |
| movu m5, [srcq] ;C |
| punpcklbw m2, m3, m5 ;A B next iter |
| punpckhbw m3, m5 ;A B next iter |
| mova tmp0, m2 ;store to stack |
| mova tmp1, m3 ;store to stack |
| movu m7, [srcq + sstrideq] ;D |
| lea srcq, [srcq + sstrideq * 2] |
| punpcklbw m4, m5, m7 ;C D |
| punpckhbw m5, m7 ;C D |
| movu m9, [srcq] ;E |
| punpcklbw m6, m7, m9 ;C D next iter |
| punpckhbw m7, m9 ;C D next iter |
| movu m11, [srcq + sstrideq] ;F |
| lea srcq, [srcq + sstrideq * 2] |
| punpcklbw m8, m9, m11 ;E F |
| punpckhbw m9, m11 ;E F |
| movu m2, [srcq] ;G |
| punpcklbw m10, m11, m2 ;E F next iter |
| punpckhbw m11, m2 ;E F next iter |
| |
| .loop: |
| ;Do two rows at once |
| pmaddubsw m13, m0, k0k1 |
| mova m0, m4 |
| pmaddubsw m14, m8, k4k5 |
| pmaddubsw m15, m4, k2k3 |
| mova m4, m8 |
| paddsw m13, m14 |
| movu m3, [srcq + sstrideq] ;H |
| lea srcq, [srcq + sstrideq * 2] |
| punpcklbw m14, m2, m3 ;G H |
| mova m8, m14 |
| pmaddubsw m14, k6k7 |
| paddsw m15, m14 |
| paddsw m13, m15 |
| paddsw m13, krd |
| psraw m13, 7 |
| |
| pmaddubsw m14, m1, k0k1 |
| pmaddubsw m1, m9, k4k5 |
| pmaddubsw m15, m5, k2k3 |
| paddsw m14, m1 |
| mova m1, m5 |
| mova m5, m9 |
| punpckhbw m2, m3 ;G H |
| mova m9, m2 |
| pmaddubsw m2, k6k7 |
| paddsw m15, m2 |
| paddsw m14, m15 |
| paddsw m14, krd |
| psraw m14, 7 |
| packuswb m13, m14 |
| %ifidn %1, v8_avg |
| pavgb m13, [dstq] |
| %endif |
| mova [dstq], m13 |
| |
| ; next iter |
| pmaddubsw m15, tmp0, k0k1 |
| pmaddubsw m14, m10, k4k5 |
| pmaddubsw m13, m6, k2k3 |
| paddsw m15, m14 |
| mova tmp0, m6 |
| mova m6, m10 |
| movu m2, [srcq] ;G next iter |
| punpcklbw m14, m3, m2 ;G H next iter |
| mova m10, m14 |
| pmaddubsw m14, k6k7 |
| paddsw m13, m14 |
| paddsw m15, m13 |
| paddsw m15, krd |
| psraw m15, 7 |
| |
| pmaddubsw m14, tmp1, k0k1 |
| mova tmp1, m7 |
| pmaddubsw m13, m7, k2k3 |
| mova m7, m11 |
| pmaddubsw m11, k4k5 |
| paddsw m14, m11 |
| punpckhbw m3, m2 ;G H next iter |
| mova m11, m3 |
| pmaddubsw m3, k6k7 |
| paddsw m13, m3 |
| paddsw m14, m13 |
| paddsw m14, krd |
| psraw m14, 7 |
| packuswb m15, m14 |
| %ifidn %1, v8_avg |
| pavgb m15, [dstq + dstrideq] |
| %endif |
| mova [dstq + dstrideq], m15 |
| lea dstq, [dstq + dstrideq * 2] |
| sub heightd, 2 |
| jg .loop |
| |
| ; Do last row if output_height is odd |
| jne .done |
| |
| movu m3, [srcq + sstrideq] ;H |
| punpcklbw m6, m2, m3 ;G H |
| punpckhbw m2, m3 ;G H |
| pmaddubsw m0, k0k1 |
| pmaddubsw m1, k0k1 |
| pmaddubsw m4, k2k3 |
| pmaddubsw m5, k2k3 |
| pmaddubsw m8, k4k5 |
| pmaddubsw m9, k4k5 |
| pmaddubsw m6, k6k7 |
| pmaddubsw m2, k6k7 |
| paddsw m0, m8 |
| paddsw m1, m9 |
| paddsw m4, m6 |
| paddsw m5, m2 |
| paddsw m0, m4 |
| paddsw m1, m5 |
| paddsw m0, krd |
| paddsw m1, krd |
| psraw m0, 7 |
| psraw m1, 7 |
| packuswb m0, m1 |
| %ifidn %1, v8_avg |
| pavgb m0, [dstq] |
| %endif |
| mova [dstq], m0 |
| |
| .done: |
| REP_RET |
| |
| %endif ; ARCH_X86_64 |
| |
| %endm |
| |
| INIT_XMM ssse3 |
| SUBPIX_VFILTER16 v8 |
| SUBPIX_VFILTER16 v8_avg |
| SUBPIX_VFILTER v8, 8 |
| SUBPIX_VFILTER v8_avg, 8 |
| SUBPIX_VFILTER v8, 4 |
| SUBPIX_VFILTER v8_avg, 4 |
| |
| %if (ARCH_X86 || X86_SUBPIX_VFILTER_PREFER_SLOW_CELERON) && \ |
| CONFIG_LOOP_RESTORATION |
| SUBPIX_VFILTER16 v8_add_src |
| SUBPIX_VFILTER v8_add_src, 8 |
| SUBPIX_VFILTER v8_add_src, 4 |
| %endif |
