;
;  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
;
;  Use of this source code is governed by a BSD-style license
;  that can be found in the LICENSE file in the root of the source
;  tree. An additional intellectual property rights grant can be found
;  in the file PATENTS.  All contributing project authors may
;  be found in the AUTHORS file in the root of the source tree.
;


%include "vpx_ports/x86_abi_support.asm"

%define xmm_filter_shift            7

;unsigned int vp8_get_mb_ss_sse2
;(
;    short *src_ptr
;)
global sym(vp8_get_mb_ss_sse2)
sym(vp8_get_mb_ss_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 1
    GET_GOT     rbx
    push rsi
    push rdi
    sub         rsp, 16
    ; end prolog


        mov         rax, arg(0) ;[src_ptr]
        mov         rcx, 8
        pxor        xmm4, xmm4

NEXTROW:
        movdqa      xmm0, [rax]
        movdqa      xmm1, [rax+16]
        movdqa      xmm2, [rax+32]
        movdqa      xmm3, [rax+48]
        pmaddwd     xmm0, xmm0
        pmaddwd     xmm1, xmm1
        pmaddwd     xmm2, xmm2
        pmaddwd     xmm3, xmm3

        paddd       xmm0, xmm1
        paddd       xmm2, xmm3
        paddd       xmm4, xmm0
        paddd       xmm4, xmm2

        add         rax, 0x40
        dec         rcx
        ja          NEXTROW

        movdqa      xmm3,xmm4
        psrldq      xmm4,8
        paddd       xmm4,xmm3
        movdqa      xmm3,xmm4
        psrldq      xmm4,4
        paddd       xmm4,xmm3
        movq        rax,xmm4


    ; begin epilog
    add rsp, 16
    pop rdi
    pop rsi
    RESTORE_GOT
    UNSHADOW_ARGS
    pop         rbp
    ret


;unsigned int vp8_get16x16var_sse2
;(
;    unsigned char   *  src_ptr,
;    int             source_stride,
;    unsigned char   *  ref_ptr,
;    int             recon_stride,
;    unsigned int    *  SSE,
;    int             *  Sum
;)
global sym(vp8_get16x16var_sse2)
sym(vp8_get16x16var_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    push rbx
    push rsi
    push rdi
    ; end prolog

        mov         rsi,            arg(0) ;[src_ptr]
        mov         rdi,            arg(2) ;[ref_ptr]

        movsxd      rax,            DWORD PTR arg(1) ;[source_stride]
        movsxd      rdx,            DWORD PTR arg(3) ;[recon_stride]

        ; Prefetch data
        lea             rcx,    [rax+rax*2]
        prefetcht0      [rsi]
        prefetcht0      [rsi+rax]
        prefetcht0      [rsi+rax*2]
        prefetcht0      [rsi+rcx]
        lea             rbx,    [rsi+rax*4]
        prefetcht0      [rbx]
        prefetcht0      [rbx+rax]
        prefetcht0      [rbx+rax*2]
        prefetcht0      [rbx+rcx]

        lea             rcx,    [rdx+rdx*2]
        prefetcht0      [rdi]
        prefetcht0      [rdi+rdx]
        prefetcht0      [rdi+rdx*2]
        prefetcht0      [rdi+rcx]
        lea             rbx,    [rdi+rdx*4]
        prefetcht0      [rbx]
        prefetcht0      [rbx+rdx]
        prefetcht0      [rbx+rdx*2]
        prefetcht0      [rbx+rcx]

        pxor        xmm0,           xmm0                        ; clear xmm0 for unpack
        pxor        xmm7,           xmm7                        ; clear xmm7 for accumulating diffs

        pxor        xmm6,           xmm6                        ; clear xmm6 for accumulating sse
        mov         rcx,            16

var16loop:
        movdqu      xmm1,           XMMWORD PTR [rsi]
        movdqu      xmm2,           XMMWORD PTR [rdi]

        prefetcht0      [rsi+rax*8]
        prefetcht0      [rdi+rdx*8]

        movdqa      xmm3,           xmm1
        movdqa      xmm4,           xmm2


        punpcklbw   xmm1,           xmm0
        punpckhbw   xmm3,           xmm0

        punpcklbw   xmm2,           xmm0
        punpckhbw   xmm4,           xmm0


        psubw       xmm1,           xmm2
        psubw       xmm3,           xmm4

        paddw       xmm7,           xmm1
        pmaddwd     xmm1,           xmm1

        paddw       xmm7,           xmm3
        pmaddwd     xmm3,           xmm3

        paddd       xmm6,           xmm1
        paddd       xmm6,           xmm3

        add         rsi,            rax
        add         rdi,            rdx

        sub         rcx,            1
        jnz         var16loop


        movdqa      xmm1,           xmm6
        pxor        xmm6,           xmm6

        pxor        xmm5,           xmm5
        punpcklwd   xmm6,           xmm7

        punpckhwd   xmm5,           xmm7
        psrad       xmm5,           16

        psrad       xmm6,           16
        paddd       xmm6,           xmm5

        movdqa      xmm2,           xmm1
        punpckldq   xmm1,           xmm0

        punpckhdq   xmm2,           xmm0
        movdqa      xmm7,           xmm6

        paddd       xmm1,           xmm2
        punpckldq   xmm6,           xmm0

        punpckhdq   xmm7,           xmm0
        paddd       xmm6,           xmm7

        movdqa      xmm2,           xmm1
        movdqa      xmm7,           xmm6

        psrldq      xmm1,           8
        psrldq      xmm6,           8

        paddd       xmm7,           xmm6
        paddd       xmm1,           xmm2

        mov         rax,            arg(5) ;[Sum]
        mov         rdi,            arg(4) ;[SSE]

        movd DWORD PTR [rax],       xmm7
        movd DWORD PTR [rdi],       xmm1


    ; begin epilog
    pop rdi
    pop rsi
    pop rbx
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret


;unsigned int vp8_get8x8var_sse2
;(
;    unsigned char   *  src_ptr,
;    int             source_stride,
;    unsigned char   *  ref_ptr,
;    int             recon_stride,
;    unsigned int    *  SSE,
;    int             *  Sum
;)
global sym(vp8_get8x8var_sse2)
sym(vp8_get8x8var_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    sub         rsp, 16
    ; end prolog

        mov         rsi,            arg(0) ;[src_ptr]
        mov         rdi,            arg(2) ;[ref_ptr]

        movsxd      rax,            DWORD PTR arg(1) ;[source_stride]
        movsxd      rdx,            DWORD PTR arg(3) ;[recon_stride]

        pxor        xmm0,           xmm0                        ; clear xmm0 for unpack
        pxor        xmm7,           xmm7                        ; clear xmm7 for accumulating diffs

        movq        xmm1,           QWORD PTR [rsi]
        movq        xmm2,           QWORD PTR [rdi]

        punpcklbw   xmm1,           xmm0
        punpcklbw   xmm2,           xmm0

        psubsw      xmm1,           xmm2
        paddw       xmm7,           xmm1

        pmaddwd     xmm1,           xmm1

        movq        xmm2,           QWORD PTR[rsi + rax]
        movq        xmm3,           QWORD PTR[rdi + rdx]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2


        movq        xmm2,           QWORD PTR[rsi + rax * 2]
        movq        xmm3,           QWORD PTR[rdi + rdx * 2]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2


        lea         rsi,            [rsi + rax * 2]
        lea         rdi,            [rdi + rdx * 2]
        movq        xmm2,           QWORD PTR[rsi + rax]
        movq        xmm3,           QWORD PTR[rdi + rdx]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2

        movq        xmm2,           QWORD PTR[rsi + rax *2]
        movq        xmm3,           QWORD PTR[rdi + rdx *2]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2


        lea         rsi,            [rsi + rax * 2]
        lea         rdi,            [rdi + rdx * 2]


        movq        xmm2,           QWORD PTR[rsi + rax]
        movq        xmm3,           QWORD PTR[rdi + rdx]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2

        movq        xmm2,           QWORD PTR[rsi + rax *2]
        movq        xmm3,           QWORD PTR[rdi + rdx *2]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2


        lea         rsi,            [rsi + rax * 2]
        lea         rdi,            [rdi + rdx * 2]

        movq        xmm2,           QWORD PTR[rsi + rax]
        movq        xmm3,           QWORD PTR[rdi + rdx]

        punpcklbw   xmm2,           xmm0
        punpcklbw   xmm3,           xmm0

        psubsw      xmm2,           xmm3
        paddw       xmm7,           xmm2

        pmaddwd     xmm2,           xmm2
        paddd       xmm1,           xmm2


        movdqa      xmm6,           xmm7
        punpcklwd   xmm6,           xmm0

        punpckhwd   xmm7,           xmm0
        movdqa      xmm2,           xmm1

        paddw       xmm6,           xmm7
        punpckldq   xmm1,           xmm0

        punpckhdq   xmm2,           xmm0
        movdqa      xmm7,           xmm6

        paddd       xmm1,           xmm2
        punpckldq   xmm6,           xmm0

        punpckhdq   xmm7,           xmm0
        paddw       xmm6,           xmm7

        movdqa      xmm2,           xmm1
        movdqa      xmm7,           xmm6

        psrldq      xmm1,           8
        psrldq      xmm6,           8

        paddw       xmm7,           xmm6
        paddd       xmm1,           xmm2

        mov         rax,            arg(5) ;[Sum]
        mov         rdi,            arg(4) ;[SSE]

        movq        rdx,            xmm7
        movsx       rcx,            dx

        mov  dword ptr [rax],       ecx
        movd DWORD PTR [rdi],       xmm1

    ; begin epilog
    add rsp, 16
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_filter_block2d_bil_var_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int  xoffset,
;    int  yoffset,
;    int *sum,
;    unsigned int *sumsquared;;
;
;)
global sym(vp8_filter_block2d_bil_var_sse2)
sym(vp8_filter_block2d_bil_var_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 9
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    push rbx
    ; end prolog

        pxor            xmm6,           xmm6                 ;
        pxor            xmm7,           xmm7                 ;

        lea             rsi,            [GLOBAL(xmm_bi_rd)]  ; rounding
        movdqa          xmm4,           XMMWORD PTR [rsi]

        lea             rcx,            [GLOBAL(vp8_bilinear_filters_sse2)]
        movsxd          rax,            dword ptr arg(5)     ; xoffset

        cmp             rax,            0                    ; skip first_pass filter if xoffset=0
        je              filter_block2d_bil_var_sse2_sp_only

        shl             rax,            5                    ; point to filter coeff with xoffset
        lea             rax,            [rax + rcx]          ; HFilter

        movsxd          rdx,            dword ptr arg(6)     ; yoffset

        cmp             rdx,            0                    ; skip second_pass filter if yoffset=0
        je              filter_block2d_bil_var_sse2_fp_only

        shl             rdx,            5
        lea             rdx,            [rdx + rcx]          ; VFilter

        mov             rsi,            arg(0)               ;ref_ptr
        mov             rdi,            arg(2)               ;src_ptr
        movsxd          rcx,            dword ptr arg(4)     ;Height

        pxor            xmm0,           xmm0                 ;
        movq            xmm1,           QWORD PTR [rsi]      ;
        movq            xmm3,           QWORD PTR [rsi+1]    ;

        punpcklbw       xmm1,           xmm0                 ;
        pmullw          xmm1,           [rax]                ;
        punpcklbw       xmm3,           xmm0
        pmullw          xmm3,           [rax+16]             ;

        paddw           xmm1,           xmm3                 ;
        paddw           xmm1,           xmm4                 ;
        psraw           xmm1,           xmm_filter_shift     ;
        movdqa          xmm5,           xmm1

        movsxd          rbx,            dword ptr arg(1) ;ref_pixels_per_line
        lea             rsi,            [rsi + rbx]
%if ABI_IS_32BIT=0
        movsxd          r9,             dword ptr arg(3) ;src_pixels_per_line
%endif

filter_block2d_bil_var_sse2_loop:
        movq            xmm1,           QWORD PTR [rsi]               ;
        movq            xmm3,           QWORD PTR [rsi+1]             ;

        punpcklbw       xmm1,           xmm0                 ;
        pmullw          xmm1,           [rax]               ;
        punpcklbw       xmm3,           xmm0                 ;
        pmullw          xmm3,           [rax+16]             ;

        paddw           xmm1,           xmm3                 ;
        paddw           xmm1,           xmm4               ;
        psraw           xmm1,           xmm_filter_shift    ;

        movdqa          xmm3,           xmm5                 ;
        movdqa          xmm5,           xmm1                 ;

        pmullw          xmm3,           [rdx]               ;
        pmullw          xmm1,           [rdx+16]             ;
        paddw           xmm1,           xmm3                 ;
        paddw           xmm1,           xmm4                 ;
        psraw           xmm1,           xmm_filter_shift    ;

        movq            xmm3,           QWORD PTR [rdi]               ;
        punpcklbw       xmm3,           xmm0                 ;

        psubw           xmm1,           xmm3                 ;
        paddw           xmm6,           xmm1                 ;

        pmaddwd         xmm1,           xmm1                 ;
        paddd           xmm7,           xmm1                 ;

        lea             rsi,            [rsi + rbx]          ;ref_pixels_per_line
%if ABI_IS_32BIT
        add             rdi,            dword ptr arg(3)     ;src_pixels_per_line
%else
        lea             rdi,            [rdi + r9]
%endif

        sub             rcx,            1                   ;
        jnz             filter_block2d_bil_var_sse2_loop       ;

        jmp             filter_block2d_bil_variance

filter_block2d_bil_var_sse2_sp_only:
        movsxd          rdx,            dword ptr arg(6)     ; yoffset

        cmp             rdx,            0                    ; skip all if both xoffset=0 and yoffset=0
        je              filter_block2d_bil_var_sse2_full_pixel

        shl             rdx,            5
        lea             rdx,            [rdx + rcx]          ; VFilter

        mov             rsi,            arg(0)               ;ref_ptr
        mov             rdi,            arg(2)               ;src_ptr
        movsxd          rcx,            dword ptr arg(4)     ;Height
        movsxd          rax,            dword ptr arg(1)     ;ref_pixels_per_line

        pxor            xmm0,           xmm0                 ;
        movq            xmm1,           QWORD PTR [rsi]      ;
        punpcklbw       xmm1,           xmm0                 ;

        movsxd          rbx,            dword ptr arg(3)     ;src_pixels_per_line
        lea             rsi,            [rsi + rax]

filter_block2d_bil_sp_only_loop:
        movq            xmm3,           QWORD PTR [rsi]             ;
        punpcklbw       xmm3,           xmm0                 ;
        movdqa          xmm5,           xmm3

        pmullw          xmm1,           [rdx]               ;
        pmullw          xmm3,           [rdx+16]             ;
        paddw           xmm1,           xmm3                 ;
        paddw           xmm1,           xmm4                 ;
        psraw           xmm1,           xmm_filter_shift    ;

        movq            xmm3,           QWORD PTR [rdi]               ;
        punpcklbw       xmm3,           xmm0                 ;

        psubw           xmm1,           xmm3                 ;
        paddw           xmm6,           xmm1                 ;

        pmaddwd         xmm1,           xmm1                 ;
        paddd           xmm7,           xmm1                 ;

        movdqa          xmm1,           xmm5                 ;
        lea             rsi,            [rsi + rax]          ;ref_pixels_per_line
        lea             rdi,            [rdi + rbx]          ;src_pixels_per_line

        sub             rcx,            1                   ;
        jnz             filter_block2d_bil_sp_only_loop       ;

        jmp             filter_block2d_bil_variance

filter_block2d_bil_var_sse2_full_pixel:
        mov             rsi,            arg(0)               ;ref_ptr
        mov             rdi,            arg(2)               ;src_ptr
        movsxd          rcx,            dword ptr arg(4)     ;Height
        movsxd          rax,            dword ptr arg(1)     ;ref_pixels_per_line
        movsxd          rbx,            dword ptr arg(3)     ;src_pixels_per_line
        pxor            xmm0,           xmm0                 ;

filter_block2d_bil_full_pixel_loop:
        movq            xmm1,           QWORD PTR [rsi]               ;
        punpcklbw       xmm1,           xmm0                 ;

        movq            xmm2,           QWORD PTR [rdi]               ;
        punpcklbw       xmm2,           xmm0                 ;

        psubw           xmm1,           xmm2                 ;
        paddw           xmm6,           xmm1                 ;

        pmaddwd         xmm1,           xmm1                 ;
        paddd           xmm7,           xmm1                 ;

        lea             rsi,            [rsi + rax]          ;ref_pixels_per_line
        lea             rdi,            [rdi + rbx]          ;src_pixels_per_line

        sub             rcx,            1                   ;
        jnz             filter_block2d_bil_full_pixel_loop       ;

        jmp             filter_block2d_bil_variance

filter_block2d_bil_var_sse2_fp_only:
        mov             rsi,            arg(0)               ;ref_ptr
        mov             rdi,            arg(2)               ;src_ptr
        movsxd          rcx,            dword ptr arg(4)     ;Height
        movsxd          rdx,            dword ptr arg(1)     ;ref_pixels_per_line

        pxor            xmm0,           xmm0                 ;
        movsxd          rbx,            dword ptr arg(3)     ;src_pixels_per_line

filter_block2d_bil_fp_only_loop:
        movq            xmm1,           QWORD PTR [rsi]       ;
        movq            xmm3,           QWORD PTR [rsi+1]     ;

        punpcklbw       xmm1,           xmm0                 ;
        pmullw          xmm1,           [rax]               ;
        punpcklbw       xmm3,           xmm0                 ;
        pmullw          xmm3,           [rax+16]             ;

        paddw           xmm1,           xmm3                 ;
        paddw           xmm1,           xmm4  ;
        psraw           xmm1,           xmm_filter_shift    ;

        movq            xmm3,           QWORD PTR [rdi]     ;
        punpcklbw       xmm3,           xmm0                 ;

        psubw           xmm1,           xmm3                 ;
        paddw           xmm6,           xmm1                 ;

        pmaddwd         xmm1,           xmm1                 ;
        paddd           xmm7,           xmm1                 ;
        lea             rsi,            [rsi + rdx]
        lea             rdi,            [rdi + rbx]          ;src_pixels_per_line

        sub             rcx,            1                   ;
        jnz             filter_block2d_bil_fp_only_loop       ;

        jmp             filter_block2d_bil_variance

filter_block2d_bil_variance:
        movdq2q         mm6,            xmm6                ;
        movdq2q         mm7,            xmm7                ;

        psrldq          xmm6,           8
        psrldq          xmm7,           8

        movdq2q         mm2,            xmm6
        movdq2q         mm3,            xmm7

        paddw           mm6,            mm2
        paddd           mm7,            mm3

        pxor            mm3,            mm3                 ;
        pxor            mm2,            mm2                 ;

        punpcklwd       mm2,            mm6                 ;
        punpckhwd       mm3,            mm6                 ;

        paddd           mm2,            mm3                 ;
        movq            mm6,            mm2                 ;

        psrlq           mm6,            32                  ;
        paddd           mm2,            mm6                 ;

        psrad           mm2,            16                  ;
        movq            mm4,            mm7                 ;

        psrlq           mm4,            32                  ;
        paddd           mm4,            mm7                 ;

        mov             rsi,            arg(7) ; sum
        mov             rdi,            arg(8) ; sumsquared

        movd            [rsi],          mm2    ; xsum
        movd            [rdi],          mm4    ; xxsum

    ; begin epilog
    pop rbx
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret


;void vp8_half_horiz_vert_variance8x_h_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int *sum,
;    unsigned int *sumsquared
;)
global sym(vp8_half_horiz_vert_variance8x_h_sse2)
sym(vp8_half_horiz_vert_variance8x_h_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 7
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    ; end prolog

%if ABI_IS_32BIT=0
    movsxd          r8, dword ptr arg(1) ;ref_pixels_per_line
    movsxd          r9, dword ptr arg(3) ;src_pixels_per_line
%endif

        pxor            xmm6,           xmm6                ;  error accumulator
        pxor            xmm7,           xmm7                ;  sse eaccumulator
        mov             rsi,            arg(0) ;ref_ptr              ;

        mov             rdi,            arg(2) ;src_ptr              ;
        movsxd          rcx,            dword ptr arg(4) ;Height              ;
        movsxd          rax,            dword ptr arg(1) ;ref_pixels_per_line

        pxor            xmm0,           xmm0                ;

        movq            xmm5,           QWORD PTR [rsi]     ;  xmm5 = s0,s1,s2..s8
        movq            xmm3,           QWORD PTR [rsi+1]   ;  xmm3 = s1,s2,s3..s9
        pavgb           xmm5,           xmm3                ;  xmm5 = avg(xmm1,xmm3) horizontal line 1

%if ABI_IS_32BIT
        add             rsi,            dword ptr arg(1) ;ref_pixels_per_line    ;  next source
%else
        add             rsi, r8
%endif

vp8_half_horiz_vert_variance8x_h_1:

        movq            xmm1,           QWORD PTR [rsi]     ;
        movq            xmm2,           QWORD PTR [rsi+1]   ;
        pavgb           xmm1,           xmm2                ;  xmm1 = avg(xmm1,xmm3) horizontal line i+1

        pavgb           xmm5,           xmm1                ;  xmm = vertical average of the above
        punpcklbw       xmm5,           xmm0                ;  xmm5 = words of above

        movq            xmm3,           QWORD PTR [rdi]     ;  xmm3 = d0,d1,d2..d8
        punpcklbw       xmm3,           xmm0                ;  xmm3 = words of above

        psubw           xmm5,           xmm3                ;  xmm5 -= xmm3
        paddw           xmm6,           xmm5                ;  xmm6 += accumulated column differences
        pmaddwd         xmm5,           xmm5                ;  xmm5 *= xmm5
        paddd           xmm7,           xmm5                ;  xmm7 += accumulated square column differences

        movdqa          xmm5,           xmm1                ;  save xmm1 for use on the next row

%if ABI_IS_32BIT
        add             esi,            dword ptr arg(1) ;ref_pixels_per_line    ;  next source
        add             edi,            dword ptr arg(3) ;src_pixels_per_line    ;  next destination
%else
        add             rsi, r8
        add             rdi, r9
%endif

        sub             rcx,            1                   ;
        jnz             vp8_half_horiz_vert_variance8x_h_1     ;

        movdq2q         mm6,            xmm6                ;
        movdq2q         mm7,            xmm7                ;

        psrldq          xmm6,           8
        psrldq          xmm7,           8

        movdq2q         mm2,            xmm6
        movdq2q         mm3,            xmm7

        paddw           mm6,            mm2
        paddd           mm7,            mm3

        pxor            mm3,            mm3                 ;
        pxor            mm2,            mm2                 ;

        punpcklwd       mm2,            mm6                 ;
        punpckhwd       mm3,            mm6                 ;

        paddd           mm2,            mm3                 ;
        movq            mm6,            mm2                 ;

        psrlq           mm6,            32                  ;
        paddd           mm2,            mm6                 ;

        psrad           mm2,            16                  ;
        movq            mm4,            mm7                 ;

        psrlq           mm4,            32                  ;
        paddd           mm4,            mm7                 ;

        mov             rsi,            arg(5) ; sum
        mov             rdi,            arg(6) ; sumsquared

        movd            [rsi],          mm2                 ;
        movd            [rdi],          mm4                 ;


    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_half_horiz_vert_variance16x_h_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int *sum,
;    unsigned int *sumsquared
;)
global sym(vp8_half_horiz_vert_variance16x_h_sse2)
sym(vp8_half_horiz_vert_variance16x_h_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 7
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    ; end prolog

        pxor            xmm6,           xmm6                ;  error accumulator
        pxor            xmm7,           xmm7                ;  sse eaccumulator
        mov             rsi,            arg(0) ;ref_ptr              ;

        mov             rdi,            arg(2) ;src_ptr              ;
        movsxd          rcx,            dword ptr arg(4) ;Height              ;
        movsxd          rax,            dword ptr arg(1) ;ref_pixels_per_line
        movsxd          rdx,            dword ptr arg(3)    ;src_pixels_per_line

        pxor            xmm0,           xmm0                ;

        movdqu          xmm5,           XMMWORD PTR [rsi]
        movdqu          xmm3,           XMMWORD PTR [rsi+1]
        pavgb           xmm5,           xmm3                ;  xmm5 = avg(xmm1,xmm3) horizontal line 1

        lea             rsi,            [rsi + rax]

vp8_half_horiz_vert_variance16x_h_1:
        movdqu          xmm1,           XMMWORD PTR [rsi]     ;
        movdqu          xmm2,           XMMWORD PTR [rsi+1]   ;
        pavgb           xmm1,           xmm2                ;  xmm1 = avg(xmm1,xmm3) horizontal line i+1

        pavgb           xmm5,           xmm1                ;  xmm = vertical average of the above

        movdqa          xmm4,           xmm5
        punpcklbw       xmm5,           xmm0                ;  xmm5 = words of above
        punpckhbw       xmm4,           xmm0

        movq            xmm3,           QWORD PTR [rdi]     ;  xmm3 = d0,d1,d2..d7
        punpcklbw       xmm3,           xmm0                ;  xmm3 = words of above
        psubw           xmm5,           xmm3                ;  xmm5 -= xmm3

        movq            xmm3,           QWORD PTR [rdi+8]
        punpcklbw       xmm3,           xmm0
        psubw           xmm4,           xmm3

        paddw           xmm6,           xmm5                ;  xmm6 += accumulated column differences
        paddw           xmm6,           xmm4
        pmaddwd         xmm5,           xmm5                ;  xmm5 *= xmm5
        pmaddwd         xmm4,           xmm4
        paddd           xmm7,           xmm5                ;  xmm7 += accumulated square column differences
        paddd           xmm7,           xmm4

        movdqa          xmm5,           xmm1                ;  save xmm1 for use on the next row

        lea             rsi,            [rsi + rax]
        lea             rdi,            [rdi + rdx]

        sub             rcx,            1                   ;
        jnz             vp8_half_horiz_vert_variance16x_h_1     ;

        pxor        xmm1,           xmm1
        pxor        xmm5,           xmm5

        punpcklwd   xmm0,           xmm6
        punpckhwd   xmm1,           xmm6
        psrad       xmm0,           16
        psrad       xmm1,           16
        paddd       xmm0,           xmm1
        movdqa      xmm1,           xmm0

        movdqa      xmm6,           xmm7
        punpckldq   xmm6,           xmm5
        punpckhdq   xmm7,           xmm5
        paddd       xmm6,           xmm7

        punpckldq   xmm0,           xmm5
        punpckhdq   xmm1,           xmm5
        paddd       xmm0,           xmm1

        movdqa      xmm7,           xmm6
        movdqa      xmm1,           xmm0

        psrldq      xmm7,           8
        psrldq      xmm1,           8

        paddd       xmm6,           xmm7
        paddd       xmm0,           xmm1

        mov         rsi,            arg(5) ;[Sum]
        mov         rdi,            arg(6) ;[SSE]

        movd        [rsi],       xmm0
        movd        [rdi],       xmm6

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret


;void vp8_half_vert_variance8x_h_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int *sum,
;    unsigned int *sumsquared
;)
global sym(vp8_half_vert_variance8x_h_sse2)
sym(vp8_half_vert_variance8x_h_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 7
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    ; end prolog

%if ABI_IS_32BIT=0
    movsxd          r8, dword ptr arg(1) ;ref_pixels_per_line
    movsxd          r9, dword ptr arg(3) ;src_pixels_per_line
%endif

        pxor            xmm6,           xmm6                ;  error accumulator
        pxor            xmm7,           xmm7                ;  sse eaccumulator
        mov             rsi,            arg(0) ;ref_ptr              ;

        mov             rdi,            arg(2) ;src_ptr              ;
        movsxd          rcx,            dword ptr arg(4) ;Height              ;
        movsxd          rax,            dword ptr arg(1) ;ref_pixels_per_line

        pxor            xmm0,           xmm0                ;
vp8_half_vert_variance8x_h_1:
        movq            xmm5,           QWORD PTR [rsi]     ;  xmm5 = s0,s1,s2..s8
        movq            xmm3,           QWORD PTR [rsi+rax] ;  xmm3 = s1,s2,s3..s9

        pavgb           xmm5,           xmm3                ;  xmm5 = avg(xmm1,xmm3)
        punpcklbw       xmm5,           xmm0                ;  xmm5 = words of above

        movq            xmm3,           QWORD PTR [rdi]     ;  xmm3 = d0,d1,d2..d8
        punpcklbw       xmm3,           xmm0                ;  xmm3 = words of above

        psubw           xmm5,           xmm3                ;  xmm5 -= xmm3
        paddw           xmm6,           xmm5                ;  xmm6 += accumulated column differences
        pmaddwd         xmm5,           xmm5                ;  xmm5 *= xmm5
        paddd           xmm7,           xmm5                ;  xmm7 += accumulated square column differences

%if ABI_IS_32BIT
        add             esi,            dword ptr arg(1) ;ref_pixels_per_line    ;  next source
        add             edi,            dword ptr arg(3) ;src_pixels_per_line    ;  next destination
%else
        add             rsi, r8
        add             rdi, r9
%endif

        sub             rcx,            1                   ;
        jnz             vp8_half_vert_variance8x_h_1          ;

        movdq2q         mm6,            xmm6                ;
        movdq2q         mm7,            xmm7                ;

        psrldq          xmm6,           8
        psrldq          xmm7,           8

        movdq2q         mm2,            xmm6
        movdq2q         mm3,            xmm7

        paddw           mm6,            mm2
        paddd           mm7,            mm3

        pxor            mm3,            mm3                 ;
        pxor            mm2,            mm2                 ;

        punpcklwd       mm2,            mm6                 ;
        punpckhwd       mm3,            mm6                 ;

        paddd           mm2,            mm3                 ;
        movq            mm6,            mm2                 ;

        psrlq           mm6,            32                  ;
        paddd           mm2,            mm6                 ;

        psrad           mm2,            16                  ;
        movq            mm4,            mm7                 ;

        psrlq           mm4,            32                  ;
        paddd           mm4,            mm7                 ;

        mov             rsi,            arg(5) ; sum
        mov             rdi,            arg(6) ; sumsquared

        movd            [rsi],          mm2                 ;
        movd            [rdi],          mm4                 ;


    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_half_vert_variance16x_h_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int *sum,
;    unsigned int *sumsquared
;)
global sym(vp8_half_vert_variance16x_h_sse2)
sym(vp8_half_vert_variance16x_h_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 7
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    ; end prolog

        pxor            xmm6,           xmm6                ;  error accumulator
        pxor            xmm7,           xmm7                ;  sse eaccumulator
        mov             rsi,            arg(0)              ;ref_ptr

        mov             rdi,            arg(2)              ;src_ptr
        movsxd          rcx,            dword ptr arg(4)    ;Height
        movsxd          rax,            dword ptr arg(1)    ;ref_pixels_per_line
        movsxd          rdx,            dword ptr arg(3)    ;src_pixels_per_line

        movdqu          xmm5,           XMMWORD PTR [rsi]
        lea             rsi,            [rsi + rax          ]
        pxor            xmm0,           xmm0

vp8_half_vert_variance16x_h_1:
        movdqu          xmm3,           XMMWORD PTR [rsi]

        pavgb           xmm5,           xmm3                ;  xmm5 = avg(xmm1,xmm3)
        movdqa          xmm4,           xmm5
        punpcklbw       xmm5,           xmm0
        punpckhbw       xmm4,           xmm0

        movq            xmm2,           QWORD PTR [rdi]
        punpcklbw       xmm2,           xmm0
        psubw           xmm5,           xmm2
        movq            xmm2,           QWORD PTR [rdi+8]
        punpcklbw       xmm2,           xmm0
        psubw           xmm4,           xmm2

        paddw           xmm6,           xmm5                ;  xmm6 += accumulated column differences
        paddw           xmm6,           xmm4
        pmaddwd         xmm5,           xmm5                ;  xmm5 *= xmm5
        pmaddwd         xmm4,           xmm4
        paddd           xmm7,           xmm5                ;  xmm7 += accumulated square column differences
        paddd           xmm7,           xmm4

        movdqa          xmm5,           xmm3

        lea             rsi,            [rsi + rax]
        lea             rdi,            [rdi + rdx]

        sub             rcx,            1
        jnz             vp8_half_vert_variance16x_h_1

        pxor        xmm1,           xmm1
        pxor        xmm5,           xmm5

        punpcklwd   xmm0,           xmm6
        punpckhwd   xmm1,           xmm6
        psrad       xmm0,           16
        psrad       xmm1,           16
        paddd       xmm0,           xmm1
        movdqa      xmm1,           xmm0

        movdqa      xmm6,           xmm7
        punpckldq   xmm6,           xmm5
        punpckhdq   xmm7,           xmm5
        paddd       xmm6,           xmm7

        punpckldq   xmm0,           xmm5
        punpckhdq   xmm1,           xmm5
        paddd       xmm0,           xmm1

        movdqa      xmm7,           xmm6
        movdqa      xmm1,           xmm0

        psrldq      xmm7,           8
        psrldq      xmm1,           8

        paddd       xmm6,           xmm7
        paddd       xmm0,           xmm1

        mov         rsi,            arg(5) ;[Sum]
        mov         rdi,            arg(6) ;[SSE]

        movd        [rsi],       xmm0
        movd        [rdi],       xmm6

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret


;void vp8_half_horiz_variance8x_h_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int *sum,
;    unsigned int *sumsquared
;)
global sym(vp8_half_horiz_variance8x_h_sse2)
sym(vp8_half_horiz_variance8x_h_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 7
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    ; end prolog

%if ABI_IS_32BIT=0
    movsxd          r8, dword ptr arg(1) ;ref_pixels_per_line
    movsxd          r9, dword ptr arg(3) ;src_pixels_per_line
%endif

        pxor            xmm6,           xmm6                ;  error accumulator
        pxor            xmm7,           xmm7                ;  sse eaccumulator
        mov             rsi,            arg(0) ;ref_ptr              ;

        mov             rdi,            arg(2) ;src_ptr              ;
        movsxd          rcx,            dword ptr arg(4) ;Height              ;

        pxor            xmm0,           xmm0                ;
vp8_half_horiz_variance8x_h_1:
        movq            xmm5,           QWORD PTR [rsi]     ;  xmm5 = s0,s1,s2..s8
        movq            xmm3,           QWORD PTR [rsi+1]   ;  xmm3 = s1,s2,s3..s9

        pavgb           xmm5,           xmm3                ;  xmm5 = avg(xmm1,xmm3)
        punpcklbw       xmm5,           xmm0                ;  xmm5 = words of above

        movq            xmm3,           QWORD PTR [rdi]     ;  xmm3 = d0,d1,d2..d8
        punpcklbw       xmm3,           xmm0                ;  xmm3 = words of above

        psubw           xmm5,           xmm3                ;  xmm5 -= xmm3
        paddw           xmm6,           xmm5                ;  xmm6 += accumulated column differences
        pmaddwd         xmm5,           xmm5                ;  xmm5 *= xmm5
        paddd           xmm7,           xmm5                ;  xmm7 += accumulated square column differences

%if ABI_IS_32BIT
        add             esi,            dword ptr arg(1) ;ref_pixels_per_line    ;  next source
        add             edi,            dword ptr arg(3) ;src_pixels_per_line    ;  next destination
%else
        add             rsi, r8
        add             rdi, r9
%endif
        sub             rcx,            1                   ;
        jnz             vp8_half_horiz_variance8x_h_1        ;

        movdq2q         mm6,            xmm6                ;
        movdq2q         mm7,            xmm7                ;

        psrldq          xmm6,           8
        psrldq          xmm7,           8

        movdq2q         mm2,            xmm6
        movdq2q         mm3,            xmm7

        paddw           mm6,            mm2
        paddd           mm7,            mm3

        pxor            mm3,            mm3                 ;
        pxor            mm2,            mm2                 ;

        punpcklwd       mm2,            mm6                 ;
        punpckhwd       mm3,            mm6                 ;

        paddd           mm2,            mm3                 ;
        movq            mm6,            mm2                 ;

        psrlq           mm6,            32                  ;
        paddd           mm2,            mm6                 ;

        psrad           mm2,            16                  ;
        movq            mm4,            mm7                 ;

        psrlq           mm4,            32                  ;
        paddd           mm4,            mm7                 ;

        mov             rsi,            arg(5) ; sum
        mov             rdi,            arg(6) ; sumsquared

        movd            [rsi],          mm2                 ;
        movd            [rdi],          mm4                 ;


    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_half_horiz_variance16x_h_sse2
;(
;    unsigned char *ref_ptr,
;    int ref_pixels_per_line,
;    unsigned char *src_ptr,
;    int src_pixels_per_line,
;    unsigned int Height,
;    int *sum,
;    unsigned int *sumsquared
;)
global sym(vp8_half_horiz_variance16x_h_sse2)
sym(vp8_half_horiz_variance16x_h_sse2):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 7
    SAVE_XMM 7
    GET_GOT     rbx
    push rsi
    push rdi
    ; end prolog

        pxor            xmm6,           xmm6                ;  error accumulator
        pxor            xmm7,           xmm7                ;  sse eaccumulator
        mov             rsi,            arg(0) ;ref_ptr              ;

        mov             rdi,            arg(2) ;src_ptr              ;
        movsxd          rcx,            dword ptr arg(4) ;Height              ;
        movsxd          rax,            dword ptr arg(1) ;ref_pixels_per_line
        movsxd          rdx,            dword ptr arg(3)    ;src_pixels_per_line

        pxor            xmm0,           xmm0                ;

vp8_half_horiz_variance16x_h_1:
        movdqu          xmm5,           XMMWORD PTR [rsi]     ;  xmm5 = s0,s1,s2..s15
        movdqu          xmm3,           XMMWORD PTR [rsi+1]   ;  xmm3 = s1,s2,s3..s16

        pavgb           xmm5,           xmm3                ;  xmm5 = avg(xmm1,xmm3)
        movdqa          xmm1,           xmm5
        punpcklbw       xmm5,           xmm0                ;  xmm5 = words of above
        punpckhbw       xmm1,           xmm0

        movq            xmm3,           QWORD PTR [rdi]     ;  xmm3 = d0,d1,d2..d7
        punpcklbw       xmm3,           xmm0                ;  xmm3 = words of above
        movq            xmm2,           QWORD PTR [rdi+8]
        punpcklbw       xmm2,           xmm0

        psubw           xmm5,           xmm3                ;  xmm5 -= xmm3
        psubw           xmm1,           xmm2
        paddw           xmm6,           xmm5                ;  xmm6 += accumulated column differences
        paddw           xmm6,           xmm1
        pmaddwd         xmm5,           xmm5                ;  xmm5 *= xmm5
        pmaddwd         xmm1,           xmm1
        paddd           xmm7,           xmm5                ;  xmm7 += accumulated square column differences
        paddd           xmm7,           xmm1

        lea             rsi,            [rsi + rax]
        lea             rdi,            [rdi + rdx]

        sub             rcx,            1                   ;
        jnz             vp8_half_horiz_variance16x_h_1        ;

        pxor        xmm1,           xmm1
        pxor        xmm5,           xmm5

        punpcklwd   xmm0,           xmm6
        punpckhwd   xmm1,           xmm6
        psrad       xmm0,           16
        psrad       xmm1,           16
        paddd       xmm0,           xmm1
        movdqa      xmm1,           xmm0

        movdqa      xmm6,           xmm7
        punpckldq   xmm6,           xmm5
        punpckhdq   xmm7,           xmm5
        paddd       xmm6,           xmm7

        punpckldq   xmm0,           xmm5
        punpckhdq   xmm1,           xmm5
        paddd       xmm0,           xmm1

        movdqa      xmm7,           xmm6
        movdqa      xmm1,           xmm0

        psrldq      xmm7,           8
        psrldq      xmm1,           8

        paddd       xmm6,           xmm7
        paddd       xmm0,           xmm1

        mov         rsi,            arg(5) ;[Sum]
        mov         rdi,            arg(6) ;[SSE]

        movd        [rsi],       xmm0
        movd        [rdi],       xmm6

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

SECTION_RODATA
;    short xmm_bi_rd[8] = { 64, 64, 64, 64,64, 64, 64, 64};
align 16
xmm_bi_rd:
    times 8 dw 64
align 16
vp8_bilinear_filters_sse2:
    dw 128, 128, 128, 128, 128, 128, 128, 128,  0,  0,  0,  0,  0,  0,  0,  0
    dw 112, 112, 112, 112, 112, 112, 112, 112, 16, 16, 16, 16, 16, 16, 16, 16
    dw 96, 96, 96, 96, 96, 96, 96, 96, 32, 32, 32, 32, 32, 32, 32, 32
    dw 80, 80, 80, 80, 80, 80, 80, 80, 48, 48, 48, 48, 48, 48, 48, 48
    dw 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64
    dw 48, 48, 48, 48, 48, 48, 48, 48, 80, 80, 80, 80, 80, 80, 80, 80
    dw 32, 32, 32, 32, 32, 32, 32, 32, 96, 96, 96, 96, 96, 96, 96, 96
    dw 16, 16, 16, 16, 16, 16, 16, 16, 112, 112, 112, 112, 112, 112, 112, 112