aom_dsp/arm/obmc_variance_neon.c - aom - Git at Google


            
            /*

            
             * Copyright (c) 2023, Alliance for Open Media. All rights reserved.

            
             *

            
             * This source code is subject to the terms of the BSD 2 Clause License and

            
             * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License

            
             * was not distributed with this source code in the LICENSE file, you can

            
             * obtain it at www.aomedia.org/license/software. If the Alliance for Open

            
             * Media Patent License 1.0 was not distributed with this source code in the

            
             * PATENTS file, you can obtain it at www.aomedia.org/license/patent.

            
             */


            #include <arm_neon.h>


            #include "config/aom_config.h"

            
            #include "config/aom_dsp_rtcd.h"

            
            #include "mem_neon.h"

            
            #include "sum_neon.h"


            static inline void obmc_variance_8x1_s16_neon(int16x8_t pre_s16,

            
                                                          const int32_t *wsrc,

            
                                                          const int32_t *mask,

            
                                                          int32x4_t *ssev,

            
                                                          int32x4_t *sumv) {

            
              // For 4xh and 8xh we observe it is faster to avoid the double-widening of

            
              // pre. Instead we do a single widening step and narrow the mask to 16-bits

            
              // to allow us to perform a widening multiply. Widening multiply

            
              // instructions have better throughput on some micro-architectures but for

            
              // the larger block sizes this benefit is outweighed by the additional

            
              // instruction needed to first narrow the mask vectors.


              int32x4_t wsrc_s32_lo = vld1q_s32(&wsrc[0]);

            
              int32x4_t wsrc_s32_hi = vld1q_s32(&wsrc[4]);

            
              int16x8_t mask_s16 = vuzpq_s16(vreinterpretq_s16_s32(vld1q_s32(&mask[0])),

            
                                             vreinterpretq_s16_s32(vld1q_s32(&mask[4])))

            
                                       .val[0];


              int32x4_t diff_s32_lo =

            
                  vmlsl_s16(wsrc_s32_lo, vget_low_s16(pre_s16), vget_low_s16(mask_s16));

            
              int32x4_t diff_s32_hi =

            
                  vmlsl_s16(wsrc_s32_hi, vget_high_s16(pre_s16), vget_high_s16(mask_s16));


              // ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away

            
              // from zero, however vrshrq_n_s32 rounds to nearest with ties rounded up.

            
              // This difference only affects the bit patterns at the rounding breakpoints

            
              // exactly, so we can add -1 to all negative numbers to move the breakpoint

            
              // one value across and into the correct rounding region.

            
              diff_s32_lo = vsraq_n_s32(diff_s32_lo, diff_s32_lo, 31);

            
              diff_s32_hi = vsraq_n_s32(diff_s32_hi, diff_s32_hi, 31);

            
              int32x4_t round_s32_lo = vrshrq_n_s32(diff_s32_lo, 12);

            
              int32x4_t round_s32_hi = vrshrq_n_s32(diff_s32_hi, 12);


              *sumv = vrsraq_n_s32(*sumv, diff_s32_lo, 12);

            
              *sumv = vrsraq_n_s32(*sumv, diff_s32_hi, 12);

            
              *ssev = vmlaq_s32(*ssev, round_s32_lo, round_s32_lo);

            
              *ssev = vmlaq_s32(*ssev, round_s32_hi, round_s32_hi);

            
            }


            #if AOM_ARCH_AARCH64


            // Use tbl for doing a double-width zero extension from 8->32 bits since we can

            
            // do this in one instruction rather than two (indices out of range (255 here)

            
            // are set to zero by tbl).

            
            DECLARE_ALIGNED(16, static const uint8_t, obmc_variance_permute_idx[]) = {

            
              0,  255, 255, 255, 1,  255, 255, 255, 2,  255, 255, 255, 3,  255, 255, 255,

            
              4,  255, 255, 255, 5,  255, 255, 255, 6,  255, 255, 255, 7,  255, 255, 255,

            
              8,  255, 255, 255, 9,  255, 255, 255, 10, 255, 255, 255, 11, 255, 255, 255,

            
              12, 255, 255, 255, 13, 255, 255, 255, 14, 255, 255, 255, 15, 255, 255, 255

            
            };


            static inline void obmc_variance_8x1_s32_neon(

            
                int32x4_t pre_lo, int32x4_t pre_hi, const int32_t *wsrc,

            
                const int32_t *mask, int32x4_t *ssev, int32x4_t *sumv) {

            
              int32x4_t wsrc_lo = vld1q_s32(&wsrc[0]);

            
              int32x4_t wsrc_hi = vld1q_s32(&wsrc[4]);

            
              int32x4_t mask_lo = vld1q_s32(&mask[0]);

            
              int32x4_t mask_hi = vld1q_s32(&mask[4]);


              int32x4_t diff_lo = vmlsq_s32(wsrc_lo, pre_lo, mask_lo);

            
              int32x4_t diff_hi = vmlsq_s32(wsrc_hi, pre_hi, mask_hi);


              // ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away from

            
              // zero, however vrshrq_n_s32 rounds to nearest with ties rounded up. This

            
              // difference only affects the bit patterns at the rounding breakpoints

            
              // exactly, so we can add -1 to all negative numbers to move the breakpoint

            
              // one value across and into the correct rounding region.

            
              diff_lo = vsraq_n_s32(diff_lo, diff_lo, 31);

            
              diff_hi = vsraq_n_s32(diff_hi, diff_hi, 31);

            
              int32x4_t round_lo = vrshrq_n_s32(diff_lo, 12);

            
              int32x4_t round_hi = vrshrq_n_s32(diff_hi, 12);


              *sumv = vrsraq_n_s32(*sumv, diff_lo, 12);

            
              *sumv = vrsraq_n_s32(*sumv, diff_hi, 12);

            
              *ssev = vmlaq_s32(*ssev, round_lo, round_lo);

            
              *ssev = vmlaq_s32(*ssev, round_hi, round_hi);

            
            }


            static inline void obmc_variance_large_neon(const uint8_t *pre, int pre_stride,

            
                                                        const int32_t *wsrc,

            
                                                        const int32_t *mask, int width,

            
                                                        int height, unsigned *sse,

            
                                                        int *sum) {

            
              assert(width % 16 == 0);


              // Use tbl for doing a double-width zero extension from 8->32 bits since we

            
              // can do this in one instruction rather than two.

            
              uint8x16_t pre_idx0 = vld1q_u8(&obmc_variance_permute_idx[0]);

            
              uint8x16_t pre_idx1 = vld1q_u8(&obmc_variance_permute_idx[16]);

            
              uint8x16_t pre_idx2 = vld1q_u8(&obmc_variance_permute_idx[32]);

            
              uint8x16_t pre_idx3 = vld1q_u8(&obmc_variance_permute_idx[48]);


              int32x4_t ssev = vdupq_n_s32(0);

            
              int32x4_t sumv = vdupq_n_s32(0);


              int h = height;

            
              do {

            
                int w = width;

            
                do {

            
                  uint8x16_t pre_u8 = vld1q_u8(pre);


                  int32x4_t pre_s32_lo = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx0));

            
                  int32x4_t pre_s32_hi = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx1));

            
                  obmc_variance_8x1_s32_neon(pre_s32_lo, pre_s32_hi, &wsrc[0], &mask[0],

            
                                             &ssev, &sumv);


                  pre_s32_lo = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx2));

            
                  pre_s32_hi = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx3));

            
                  obmc_variance_8x1_s32_neon(pre_s32_lo, pre_s32_hi, &wsrc[8], &mask[8],

            
                                             &ssev, &sumv);


                  wsrc += 16;

            
                  mask += 16;

            
                  pre += 16;

            
                  w -= 16;

            
                } while (w != 0);


                pre += pre_stride - width;

            
              } while (--h != 0);


              *sse = horizontal_add_s32x4(ssev);

            
              *sum = horizontal_add_s32x4(sumv);

            
            }


            #else  // !AOM_ARCH_AARCH64


            static inline void obmc_variance_large_neon(const uint8_t *pre, int pre_stride,

            
                                                        const int32_t *wsrc,

            
                                                        const int32_t *mask, int width,

            
                                                        int height, unsigned *sse,

            
                                                        int *sum) {

            
              // Non-aarch64 targets do not have a 128-bit tbl instruction, so use the

            
              // widening version of the core kernel instead.


              assert(width % 16 == 0);


              int32x4_t ssev = vdupq_n_s32(0);

            
              int32x4_t sumv = vdupq_n_s32(0);


              int h = height;

            
              do {

            
                int w = width;

            
                do {

            
                  uint8x16_t pre_u8 = vld1q_u8(pre);


                  int16x8_t pre_s16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pre_u8)));

            
                  obmc_variance_8x1_s16_neon(pre_s16, &wsrc[0], &mask[0], &ssev, &sumv);


                  pre_s16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pre_u8)));

            
                  obmc_variance_8x1_s16_neon(pre_s16, &wsrc[8], &mask[8], &ssev, &sumv);


                  wsrc += 16;

            
                  mask += 16;

            
                  pre += 16;

            
                  w -= 16;

            
                } while (w != 0);


                pre += pre_stride - width;

            
              } while (--h != 0);


              *sse = horizontal_add_s32x4(ssev);

            
              *sum = horizontal_add_s32x4(sumv);

            
            }


            #endif  // AOM_ARCH_AARCH64


            static inline void obmc_variance_neon_128xh(const uint8_t *pre, int pre_stride,

            
                                                        const int32_t *wsrc,

            
                                                        const int32_t *mask, int h,

            
                                                        unsigned *sse, int *sum) {

            
              obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 128, h, sse, sum);

            
            }


            static inline void obmc_variance_neon_64xh(const uint8_t *pre, int pre_stride,

            
                                                       const int32_t *wsrc,

            
                                                       const int32_t *mask, int h,

            
                                                       unsigned *sse, int *sum) {

            
              obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 64, h, sse, sum);

            
            }


            static inline void obmc_variance_neon_32xh(const uint8_t *pre, int pre_stride,

            
                                                       const int32_t *wsrc,

            
                                                       const int32_t *mask, int h,

            
                                                       unsigned *sse, int *sum) {

            
              obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 32, h, sse, sum);

            
            }


            static inline void obmc_variance_neon_16xh(const uint8_t *pre, int pre_stride,

            
                                                       const int32_t *wsrc,

            
                                                       const int32_t *mask, int h,

            
                                                       unsigned *sse, int *sum) {

            
              obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 16, h, sse, sum);

            
            }


            static inline void obmc_variance_neon_8xh(const uint8_t *pre, int pre_stride,

            
                                                      const int32_t *wsrc,

            
                                                      const int32_t *mask, int h,

            
                                                      unsigned *sse, int *sum) {

            
              int32x4_t ssev = vdupq_n_s32(0);

            
              int32x4_t sumv = vdupq_n_s32(0);


              do {

            
                uint8x8_t pre_u8 = vld1_u8(pre);

            
                int16x8_t pre_s16 = vreinterpretq_s16_u16(vmovl_u8(pre_u8));


                obmc_variance_8x1_s16_neon(pre_s16, wsrc, mask, &ssev, &sumv);


                pre += pre_stride;

            
                wsrc += 8;

            
                mask += 8;

            
              } while (--h != 0);


              *sse = horizontal_add_s32x4(ssev);

            
              *sum = horizontal_add_s32x4(sumv);

            
            }


            static inline void obmc_variance_neon_4xh(const uint8_t *pre, int pre_stride,

            
                                                      const int32_t *wsrc,

            
                                                      const int32_t *mask, int h,

            
                                                      unsigned *sse, int *sum) {

            
              assert(h % 2 == 0);


              int32x4_t ssev = vdupq_n_s32(0);

            
              int32x4_t sumv = vdupq_n_s32(0);


              do {

            
                uint8x8_t pre_u8 = load_unaligned_u8(pre, pre_stride);

            
                int16x8_t pre_s16 = vreinterpretq_s16_u16(vmovl_u8(pre_u8));


                obmc_variance_8x1_s16_neon(pre_s16, wsrc, mask, &ssev, &sumv);


                pre += 2 * pre_stride;

            
                wsrc += 8;

            
                mask += 8;

            
                h -= 2;

            
              } while (h != 0);


              *sse = horizontal_add_s32x4(ssev);

            
              *sum = horizontal_add_s32x4(sumv);

            
            }


            #define OBMC_VARIANCE_WXH_NEON(W, H)                                       \

            
              unsigned aom_obmc_variance##W##x##H##_neon(                              \

            
                  const uint8_t *pre, int pre_stride, const int32_t *wsrc,             \

            
                  const int32_t *mask, unsigned *sse) {                                \

            
                int sum;                                                               \

            
                obmc_variance_neon_##W##xh(pre, pre_stride, wsrc, mask, H, sse, &sum); \

            
                return *sse - (unsigned)(((int64_t)sum * sum) / (W * H));              \

            
              }


            OBMC_VARIANCE_WXH_NEON(4, 4)

            
            OBMC_VARIANCE_WXH_NEON(4, 8)

            
            OBMC_VARIANCE_WXH_NEON(8, 4)

            
            OBMC_VARIANCE_WXH_NEON(8, 8)

            
            OBMC_VARIANCE_WXH_NEON(8, 16)

            
            OBMC_VARIANCE_WXH_NEON(16, 8)

            
            OBMC_VARIANCE_WXH_NEON(16, 16)

            
            OBMC_VARIANCE_WXH_NEON(16, 32)

            
            OBMC_VARIANCE_WXH_NEON(32, 16)

            
            OBMC_VARIANCE_WXH_NEON(32, 32)

            
            OBMC_VARIANCE_WXH_NEON(32, 64)

            
            OBMC_VARIANCE_WXH_NEON(64, 32)

            
            OBMC_VARIANCE_WXH_NEON(64, 64)

            
            OBMC_VARIANCE_WXH_NEON(64, 128)

            
            OBMC_VARIANCE_WXH_NEON(128, 64)

            
            OBMC_VARIANCE_WXH_NEON(128, 128)

            
            OBMC_VARIANCE_WXH_NEON(4, 16)

            
            OBMC_VARIANCE_WXH_NEON(16, 4)

            
            OBMC_VARIANCE_WXH_NEON(8, 32)

            
            OBMC_VARIANCE_WXH_NEON(32, 8)

            
            OBMC_VARIANCE_WXH_NEON(16, 64)

            
            OBMC_VARIANCE_WXH_NEON(64, 16)
	/*
	* Copyright (c) 2023, Alliance for Open Media. All rights reserved.
	*
	* This source code is subject to the terms of the BSD 2 Clause License and
	* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
	* was not distributed with this source code in the LICENSE file, you can
	* obtain it at www.aomedia.org/license/software. If the Alliance for Open
	* Media Patent License 1.0 was not distributed with this source code in the
	* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
	*/

	#include <arm_neon.h>

	#include "config/aom_config.h"
	#include "config/aom_dsp_rtcd.h"
	#include "mem_neon.h"
	#include "sum_neon.h"

	static inline void obmc_variance_8x1_s16_neon(int16x8_t pre_s16,
	const int32_t *wsrc,
	const int32_t *mask,
	int32x4_t *ssev,
	int32x4_t *sumv) {
	// For 4xh and 8xh we observe it is faster to avoid the double-widening of
	// pre. Instead we do a single widening step and narrow the mask to 16-bits
	// to allow us to perform a widening multiply. Widening multiply
	// instructions have better throughput on some micro-architectures but for
	// the larger block sizes this benefit is outweighed by the additional
	// instruction needed to first narrow the mask vectors.

	int32x4_t wsrc_s32_lo = vld1q_s32(&wsrc[0]);
	int32x4_t wsrc_s32_hi = vld1q_s32(&wsrc[4]);
	int16x8_t mask_s16 = vuzpq_s16(vreinterpretq_s16_s32(vld1q_s32(&mask[0])),
	vreinterpretq_s16_s32(vld1q_s32(&mask[4])))
	.val[0];

	int32x4_t diff_s32_lo =
	vmlsl_s16(wsrc_s32_lo, vget_low_s16(pre_s16), vget_low_s16(mask_s16));
	int32x4_t diff_s32_hi =
	vmlsl_s16(wsrc_s32_hi, vget_high_s16(pre_s16), vget_high_s16(mask_s16));

	// ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away
	// from zero, however vrshrq_n_s32 rounds to nearest with ties rounded up.
	// This difference only affects the bit patterns at the rounding breakpoints
	// exactly, so we can add -1 to all negative numbers to move the breakpoint
	// one value across and into the correct rounding region.
	diff_s32_lo = vsraq_n_s32(diff_s32_lo, diff_s32_lo, 31);
	diff_s32_hi = vsraq_n_s32(diff_s32_hi, diff_s32_hi, 31);
	int32x4_t round_s32_lo = vrshrq_n_s32(diff_s32_lo, 12);
	int32x4_t round_s32_hi = vrshrq_n_s32(diff_s32_hi, 12);

	sumv = vrsraq_n_s32(sumv, diff_s32_lo, 12);
	sumv = vrsraq_n_s32(sumv, diff_s32_hi, 12);
	ssev = vmlaq_s32(ssev, round_s32_lo, round_s32_lo);
	ssev = vmlaq_s32(ssev, round_s32_hi, round_s32_hi);
	}

	#if AOM_ARCH_AARCH64

	// Use tbl for doing a double-width zero extension from 8->32 bits since we can
	// do this in one instruction rather than two (indices out of range (255 here)
	// are set to zero by tbl).
	DECLARE_ALIGNED(16, static const uint8_t, obmc_variance_permute_idx[]) = {
	0, 255, 255, 255, 1, 255, 255, 255, 2, 255, 255, 255, 3, 255, 255, 255,
	4, 255, 255, 255, 5, 255, 255, 255, 6, 255, 255, 255, 7, 255, 255, 255,
	8, 255, 255, 255, 9, 255, 255, 255, 10, 255, 255, 255, 11, 255, 255, 255,
	12, 255, 255, 255, 13, 255, 255, 255, 14, 255, 255, 255, 15, 255, 255, 255
	};

	static inline void obmc_variance_8x1_s32_neon(
	int32x4_t pre_lo, int32x4_t pre_hi, const int32_t *wsrc,
	const int32_t mask, int32x4_t ssev, int32x4_t *sumv) {
	int32x4_t wsrc_lo = vld1q_s32(&wsrc[0]);
	int32x4_t wsrc_hi = vld1q_s32(&wsrc[4]);
	int32x4_t mask_lo = vld1q_s32(&mask[0]);
	int32x4_t mask_hi = vld1q_s32(&mask[4]);

	int32x4_t diff_lo = vmlsq_s32(wsrc_lo, pre_lo, mask_lo);
	int32x4_t diff_hi = vmlsq_s32(wsrc_hi, pre_hi, mask_hi);

	// ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away from
	// zero, however vrshrq_n_s32 rounds to nearest with ties rounded up. This
	// difference only affects the bit patterns at the rounding breakpoints
	// exactly, so we can add -1 to all negative numbers to move the breakpoint
	// one value across and into the correct rounding region.
	diff_lo = vsraq_n_s32(diff_lo, diff_lo, 31);
	diff_hi = vsraq_n_s32(diff_hi, diff_hi, 31);
	int32x4_t round_lo = vrshrq_n_s32(diff_lo, 12);
	int32x4_t round_hi = vrshrq_n_s32(diff_hi, 12);

	sumv = vrsraq_n_s32(sumv, diff_lo, 12);
	sumv = vrsraq_n_s32(sumv, diff_hi, 12);
	ssev = vmlaq_s32(ssev, round_lo, round_lo);
	ssev = vmlaq_s32(ssev, round_hi, round_hi);
	}

	static inline void obmc_variance_large_neon(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int width,
	int height, unsigned *sse,
	int *sum) {
	assert(width % 16 == 0);

	// Use tbl for doing a double-width zero extension from 8->32 bits since we
	// can do this in one instruction rather than two.
	uint8x16_t pre_idx0 = vld1q_u8(&obmc_variance_permute_idx[0]);
	uint8x16_t pre_idx1 = vld1q_u8(&obmc_variance_permute_idx[16]);
	uint8x16_t pre_idx2 = vld1q_u8(&obmc_variance_permute_idx[32]);
	uint8x16_t pre_idx3 = vld1q_u8(&obmc_variance_permute_idx[48]);

	int32x4_t ssev = vdupq_n_s32(0);
	int32x4_t sumv = vdupq_n_s32(0);

	int h = height;
	do {
	int w = width;
	do {
	uint8x16_t pre_u8 = vld1q_u8(pre);

	int32x4_t pre_s32_lo = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx0));
	int32x4_t pre_s32_hi = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx1));
	obmc_variance_8x1_s32_neon(pre_s32_lo, pre_s32_hi, &wsrc[0], &mask[0],
	&ssev, &sumv);

	pre_s32_lo = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx2));
	pre_s32_hi = vreinterpretq_s32_u8(vqtbl1q_u8(pre_u8, pre_idx3));
	obmc_variance_8x1_s32_neon(pre_s32_lo, pre_s32_hi, &wsrc[8], &mask[8],
	&ssev, &sumv);

	wsrc += 16;
	mask += 16;
	pre += 16;
	w -= 16;
	} while (w != 0);

	pre += pre_stride - width;
	} while (--h != 0);

	*sse = horizontal_add_s32x4(ssev);
	*sum = horizontal_add_s32x4(sumv);
	}

	#else // !AOM_ARCH_AARCH64

	static inline void obmc_variance_large_neon(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int width,
	int height, unsigned *sse,
	int *sum) {
	// Non-aarch64 targets do not have a 128-bit tbl instruction, so use the
	// widening version of the core kernel instead.

	assert(width % 16 == 0);

	int32x4_t ssev = vdupq_n_s32(0);
	int32x4_t sumv = vdupq_n_s32(0);

	int h = height;
	do {
	int w = width;
	do {
	uint8x16_t pre_u8 = vld1q_u8(pre);

	int16x8_t pre_s16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pre_u8)));
	obmc_variance_8x1_s16_neon(pre_s16, &wsrc[0], &mask[0], &ssev, &sumv);

	pre_s16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pre_u8)));
	obmc_variance_8x1_s16_neon(pre_s16, &wsrc[8], &mask[8], &ssev, &sumv);

	wsrc += 16;
	mask += 16;
	pre += 16;
	w -= 16;
	} while (w != 0);

	pre += pre_stride - width;
	} while (--h != 0);

	*sse = horizontal_add_s32x4(ssev);
	*sum = horizontal_add_s32x4(sumv);
	}

	#endif // AOM_ARCH_AARCH64

	static inline void obmc_variance_neon_128xh(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int h,
	unsigned sse, int sum) {
	obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 128, h, sse, sum);
	}

	static inline void obmc_variance_neon_64xh(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int h,
	unsigned sse, int sum) {
	obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 64, h, sse, sum);
	}

	static inline void obmc_variance_neon_32xh(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int h,
	unsigned sse, int sum) {
	obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 32, h, sse, sum);
	}

	static inline void obmc_variance_neon_16xh(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int h,
	unsigned sse, int sum) {
	obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 16, h, sse, sum);
	}

	static inline void obmc_variance_neon_8xh(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int h,
	unsigned sse, int sum) {
	int32x4_t ssev = vdupq_n_s32(0);
	int32x4_t sumv = vdupq_n_s32(0);

	do {
	uint8x8_t pre_u8 = vld1_u8(pre);
	int16x8_t pre_s16 = vreinterpretq_s16_u16(vmovl_u8(pre_u8));

	obmc_variance_8x1_s16_neon(pre_s16, wsrc, mask, &ssev, &sumv);

	pre += pre_stride;
	wsrc += 8;
	mask += 8;
	} while (--h != 0);

	*sse = horizontal_add_s32x4(ssev);
	*sum = horizontal_add_s32x4(sumv);
	}

	static inline void obmc_variance_neon_4xh(const uint8_t *pre, int pre_stride,
	const int32_t *wsrc,
	const int32_t *mask, int h,
	unsigned sse, int sum) {
	assert(h % 2 == 0);

	int32x4_t ssev = vdupq_n_s32(0);
	int32x4_t sumv = vdupq_n_s32(0);

	do {
	uint8x8_t pre_u8 = load_unaligned_u8(pre, pre_stride);
	int16x8_t pre_s16 = vreinterpretq_s16_u16(vmovl_u8(pre_u8));

	obmc_variance_8x1_s16_neon(pre_s16, wsrc, mask, &ssev, &sumv);

	pre += 2 * pre_stride;
	wsrc += 8;
	mask += 8;
	h -= 2;
	} while (h != 0);

	*sse = horizontal_add_s32x4(ssev);
	*sum = horizontal_add_s32x4(sumv);
	}

	#define OBMC_VARIANCE_WXH_NEON(W, H) \
	unsigned aom_obmc_variance##W##x##H##_neon( \
	const uint8_t pre, int pre_stride, const int32_t wsrc, \
	const int32_t mask, unsigned sse) { \
	int sum; \
	obmc_variance_neon_##W##xh(pre, pre_stride, wsrc, mask, H, sse, &sum); \
	return sse - (unsigned)(((int64_t)sum sum) / (W * H)); \
	}

	OBMC_VARIANCE_WXH_NEON(4, 4)
	OBMC_VARIANCE_WXH_NEON(4, 8)
	OBMC_VARIANCE_WXH_NEON(8, 4)
	OBMC_VARIANCE_WXH_NEON(8, 8)
	OBMC_VARIANCE_WXH_NEON(8, 16)
	OBMC_VARIANCE_WXH_NEON(16, 8)
	OBMC_VARIANCE_WXH_NEON(16, 16)
	OBMC_VARIANCE_WXH_NEON(16, 32)
	OBMC_VARIANCE_WXH_NEON(32, 16)
	OBMC_VARIANCE_WXH_NEON(32, 32)
	OBMC_VARIANCE_WXH_NEON(32, 64)
	OBMC_VARIANCE_WXH_NEON(64, 32)
	OBMC_VARIANCE_WXH_NEON(64, 64)
	OBMC_VARIANCE_WXH_NEON(64, 128)
	OBMC_VARIANCE_WXH_NEON(128, 64)
	OBMC_VARIANCE_WXH_NEON(128, 128)
	OBMC_VARIANCE_WXH_NEON(4, 16)
	OBMC_VARIANCE_WXH_NEON(16, 4)
	OBMC_VARIANCE_WXH_NEON(8, 32)
	OBMC_VARIANCE_WXH_NEON(32, 8)
	OBMC_VARIANCE_WXH_NEON(16, 64)
	OBMC_VARIANCE_WXH_NEON(64, 16)