;
;  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
;
;  Use of this source code is governed by a BSD-style license
;  that can be found in the LICENSE file in the root of the source
;  tree. An additional intellectual property rights grant can be found
;  in the file PATENTS.  All contributing project authors may
;  be found in the AUTHORS file in the root of the source tree.
;


%include "vpx_ports/x86_abi_support.asm"

%macro PROCESS_16X2X3 1
%if %1
        movdqa          xmm0,       XMMWORD PTR [rsi]
        lddqu           xmm5,       XMMWORD PTR [rdi]
        lddqu           xmm6,       XMMWORD PTR [rdi+1]
        lddqu           xmm7,       XMMWORD PTR [rdi+2]

        psadbw          xmm5,       xmm0
        psadbw          xmm6,       xmm0
        psadbw          xmm7,       xmm0
%else
        movdqa          xmm0,       XMMWORD PTR [rsi]
        lddqu           xmm1,       XMMWORD PTR [rdi]
        lddqu           xmm2,       XMMWORD PTR [rdi+1]
        lddqu           xmm3,       XMMWORD PTR [rdi+2]

        psadbw          xmm1,       xmm0
        psadbw          xmm2,       xmm0
        psadbw          xmm3,       xmm0

        paddw           xmm5,       xmm1
        paddw           xmm6,       xmm2
        paddw           xmm7,       xmm3
%endif
        movdqa          xmm0,       XMMWORD PTR [rsi+rax]
        lddqu           xmm1,       XMMWORD PTR [rdi+rdx]
        lddqu           xmm2,       XMMWORD PTR [rdi+rdx+1]
        lddqu           xmm3,       XMMWORD PTR [rdi+rdx+2]

        lea             rsi,        [rsi+rax*2]
        lea             rdi,        [rdi+rdx*2]

        psadbw          xmm1,       xmm0
        psadbw          xmm2,       xmm0
        psadbw          xmm3,       xmm0

        paddw           xmm5,       xmm1
        paddw           xmm6,       xmm2
        paddw           xmm7,       xmm3
%endmacro

%macro PROCESS_16X2X3_OFFSET 2
%if %1
        movdqa          xmm0,       XMMWORD PTR [rsi]
        movdqa          xmm4,       XMMWORD PTR [rdi]
        movdqa          xmm7,       XMMWORD PTR [rdi+16]

        movdqa          xmm5,       xmm7
        palignr         xmm5,       xmm4,       %2

        movdqa          xmm6,       xmm7
        palignr         xmm6,       xmm4,       (%2+1)

        palignr         xmm7,       xmm4,       (%2+2)

        psadbw          xmm5,       xmm0
        psadbw          xmm6,       xmm0
        psadbw          xmm7,       xmm0
%else
        movdqa          xmm0,       XMMWORD PTR [rsi]
        movdqa          xmm4,       XMMWORD PTR [rdi]
        movdqa          xmm3,       XMMWORD PTR [rdi+16]

        movdqa          xmm1,       xmm3
        palignr         xmm1,       xmm4,       %2

        movdqa          xmm2,       xmm3
        palignr         xmm2,       xmm4,       (%2+1)

        palignr         xmm3,       xmm4,       (%2+2)

        psadbw          xmm1,       xmm0
        psadbw          xmm2,       xmm0
        psadbw          xmm3,       xmm0

        paddw           xmm5,       xmm1
        paddw           xmm6,       xmm2
        paddw           xmm7,       xmm3
%endif
        movdqa          xmm0,       XMMWORD PTR [rsi+rax]
        movdqa          xmm4,       XMMWORD PTR [rdi+rdx]
        movdqa          xmm3,       XMMWORD PTR [rdi+rdx+16]

        movdqa          xmm1,       xmm3
        palignr         xmm1,       xmm4,       %2

        movdqa          xmm2,       xmm3
        palignr         xmm2,       xmm4,       (%2+1)

        palignr         xmm3,       xmm4,       (%2+2)

        lea             rsi,        [rsi+rax*2]
        lea             rdi,        [rdi+rdx*2]

        psadbw          xmm1,       xmm0
        psadbw          xmm2,       xmm0
        psadbw          xmm3,       xmm0

        paddw           xmm5,       xmm1
        paddw           xmm6,       xmm2
        paddw           xmm7,       xmm3
%endmacro

%macro PROCESS_16X16X3_OFFSET 2
%2_aligned_by_%1:

        sub             rdi,        %1

        PROCESS_16X2X3_OFFSET 1, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1

        jmp             %2_store_off

%endmacro

%macro PROCESS_16X8X3_OFFSET 2
%2_aligned_by_%1:

        sub             rdi,        %1

        PROCESS_16X2X3_OFFSET 1, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1
        PROCESS_16X2X3_OFFSET 0, %1

        jmp             %2_store_off

%endmacro

;void int vp9_sad16x16x3_ssse3(
;    unsigned char *src_ptr,
;    int  src_stride,
;    unsigned char *ref_ptr,
;    int  ref_stride,
;    int  *results)
global sym(vp9_sad16x16x3_ssse3)
sym(vp9_sad16x16x3_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 5
    SAVE_XMM 7
    push        rsi
    push        rdi
    push        rcx
    ; end prolog

        mov             rsi,        arg(0) ;src_ptr
        mov             rdi,        arg(2) ;ref_ptr

        mov             rdx,        0xf
        and             rdx,        rdi

        jmp .vp9_sad16x16x3_ssse3_skiptable
.vp9_sad16x16x3_ssse3_jumptable:
        dd .vp9_sad16x16x3_ssse3_aligned_by_0  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_1  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_2  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_3  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_4  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_5  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_6  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_7  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_8  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_9  - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_10 - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_11 - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_12 - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_13 - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_14 - .vp9_sad16x16x3_ssse3_do_jump
        dd .vp9_sad16x16x3_ssse3_aligned_by_15 - .vp9_sad16x16x3_ssse3_do_jump
.vp9_sad16x16x3_ssse3_skiptable:

        call .vp9_sad16x16x3_ssse3_do_jump
.vp9_sad16x16x3_ssse3_do_jump:
        pop             rcx                         ; get the address of do_jump
        mov             rax,  .vp9_sad16x16x3_ssse3_jumptable - .vp9_sad16x16x3_ssse3_do_jump
        add             rax,  rcx  ; get the absolute address of vp9_sad16x16x3_ssse3_jumptable

        movsxd          rax,  dword [rax + 4*rdx]   ; get the 32 bit offset from the jumptable
        add             rcx,        rax

        movsxd          rax,        dword ptr arg(1) ;src_stride
        movsxd          rdx,        dword ptr arg(3) ;ref_stride

        jmp             rcx

        PROCESS_16X16X3_OFFSET 0,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 1,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 2,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 3,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 4,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 5,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 6,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 7,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 8,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 9,  .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 10, .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 11, .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 12, .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 13, .vp9_sad16x16x3_ssse3
        PROCESS_16X16X3_OFFSET 14, .vp9_sad16x16x3_ssse3

.vp9_sad16x16x3_ssse3_aligned_by_15:
        PROCESS_16X2X3 1
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0

.vp9_sad16x16x3_ssse3_store_off:
        mov             rdi,        arg(4) ;Results

        movq            xmm0,       xmm5
        psrldq          xmm5,       8

        paddw           xmm0,       xmm5
        movd            [rdi],      xmm0
;-
        movq            xmm0,       xmm6
        psrldq          xmm6,       8

        paddw           xmm0,       xmm6
        movd            [rdi+4],    xmm0
;-
        movq            xmm0,       xmm7
        psrldq          xmm7,       8

        paddw           xmm0,       xmm7
        movd            [rdi+8],    xmm0

    ; begin epilog
    pop         rcx
    pop         rdi
    pop         rsi
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void int vp9_sad16x8x3_ssse3(
;    unsigned char *src_ptr,
;    int  src_stride,
;    unsigned char *ref_ptr,
;    int  ref_stride,
;    int  *results)
global sym(vp9_sad16x8x3_ssse3)
sym(vp9_sad16x8x3_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 5
    SAVE_XMM 7
    push        rsi
    push        rdi
    push        rcx
    ; end prolog

        mov             rsi,        arg(0) ;src_ptr
        mov             rdi,        arg(2) ;ref_ptr

        mov             rdx,        0xf
        and             rdx,        rdi

        jmp .vp9_sad16x8x3_ssse3_skiptable
.vp9_sad16x8x3_ssse3_jumptable:
        dd .vp9_sad16x8x3_ssse3_aligned_by_0  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_1  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_2  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_3  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_4  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_5  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_6  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_7  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_8  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_9  - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_10 - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_11 - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_12 - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_13 - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_14 - .vp9_sad16x8x3_ssse3_do_jump
        dd .vp9_sad16x8x3_ssse3_aligned_by_15 - .vp9_sad16x8x3_ssse3_do_jump
.vp9_sad16x8x3_ssse3_skiptable:

        call .vp9_sad16x8x3_ssse3_do_jump
.vp9_sad16x8x3_ssse3_do_jump:
        pop             rcx                         ; get the address of do_jump
        mov             rax,  .vp9_sad16x8x3_ssse3_jumptable - .vp9_sad16x8x3_ssse3_do_jump
        add             rax,  rcx  ; get the absolute address of vp9_sad16x8x3_ssse3_jumptable

        movsxd          rax,  dword [rax + 4*rdx]   ; get the 32 bit offset from the jumptable
        add             rcx,        rax

        movsxd          rax,        dword ptr arg(1) ;src_stride
        movsxd          rdx,        dword ptr arg(3) ;ref_stride

        jmp             rcx

        PROCESS_16X8X3_OFFSET 0,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 1,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 2,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 3,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 4,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 5,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 6,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 7,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 8,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 9,  .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 10, .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 11, .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 12, .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 13, .vp9_sad16x8x3_ssse3
        PROCESS_16X8X3_OFFSET 14, .vp9_sad16x8x3_ssse3

.vp9_sad16x8x3_ssse3_aligned_by_15:

        PROCESS_16X2X3 1
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0
        PROCESS_16X2X3 0

.vp9_sad16x8x3_ssse3_store_off:
        mov             rdi,        arg(4) ;Results

        movq            xmm0,       xmm5
        psrldq          xmm5,       8

        paddw           xmm0,       xmm5
        movd            [rdi],      xmm0
;-
        movq            xmm0,       xmm6
        psrldq          xmm6,       8

        paddw           xmm0,       xmm6
        movd            [rdi+4],    xmm0
;-
        movq            xmm0,       xmm7
        psrldq          xmm7,       8

        paddw           xmm0,       xmm7
        movd            [rdi+8],    xmm0

    ; begin epilog
    pop         rcx
    pop         rdi
    pop         rsi
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret