av1/encoder/x86/frame_error_sse2.c - aom - Git at Google


            
            /*

            
             * Copyright (c) 2019, Alliance for Open Media. All rights reserved

            
             *

            
             * This source code is subject to the terms of the BSD 2 Clause License and

            
             * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License

            
             * was not distributed with this source code in the LICENSE file, you can

            
             * obtain it at www.aomedia.org/license/software. If the Alliance for Open

            
             * Media Patent License 1.0 was not distributed with this source code in the

            
             * PATENTS file, you can obtain it at www.aomedia.org/license/patent.

            
             */


            #include <emmintrin.h>


            #include "aom_dsp/x86/synonyms.h"

            
            #include "av1/common/warped_motion.h"

            
            #include "av1/encoder/global_motion.h"

            
            #include "config/av1_rtcd.h"


            #if CONFIG_AV1_HIGHBITDEPTH

            
            int64_t av1_calc_highbd_frame_error_sse2(const uint16_t *const ref,

            
                                                     int ref_stride,

            
                                                     const uint16_t *const dst,

            
                                                     int dst_stride, int p_width,

            
                                                     int p_height, int bd) {

            
              const int b = bd - 8;

            
              const __m128i shift = _mm_cvtsi32_si128(b);

            
              const __m128i bmask = _mm_set1_epi16((1 << b) - 1);

            
              const __m128i v = _mm_set1_epi16(1 << b);


              int64_t sum_error = 0;

            
              int i, j;

            
              __m128i row_error, col_error;

            
              const __m128i zero = _mm_setzero_si128();

            
              const __m128i dup_256 = _mm_set1_epi16(256);

            
              const __m128i dup_257 = _mm_set1_epi16(257);

            
              col_error = zero;

            
              for (i = 0; i < (p_height); i++) {

            
                row_error = zero;

            
                for (j = 0; j < (p_width / 16); j++) {

            
                  const __m128i ref_1 =

            
                      _mm_load_si128((__m128i *)(ref + (j * 16) + (i * ref_stride)));

            
                  const __m128i dst_1 =

            
                      _mm_load_si128((__m128i *)(dst + (j * 16) + (i * dst_stride)));

            
                  const __m128i ref_2 =

            
                      _mm_load_si128((__m128i *)(ref + (j * 16 + 8) + (i * ref_stride)));

            
                  const __m128i dst_2 =

            
                      _mm_load_si128((__m128i *)(dst + (j * 16 + 8) + (i * dst_stride)));


                  const __m128i diff_1 = _mm_sub_epi16(dst_1, ref_1);

            
                  const __m128i diff_2 = _mm_sub_epi16(dst_2, ref_2);


                  const __m128i e1_1 = _mm_sra_epi16(diff_1, shift);

            
                  const __m128i e2_1 = _mm_and_si128(diff_1, bmask);

            
                  const __m128i e1_2 = _mm_sra_epi16(diff_2, shift);

            
                  const __m128i e2_2 = _mm_and_si128(diff_2, bmask);


                  // For each 16-bit element in e1 and e2, we need to accumulate

            
                  // the value:

            
                  //   error_measure_lut[256 + e1] * (v - e2) +

            
                  //   error_measure_lut[257 + e1] * e2

            
                  // To do this, we first synthesize two 16-bit gathers, then

            
                  // interleave the factors in such a way that we can use _mm_madd_epi16

            
                  // to do eight 16x16->32 multiplies in one go

            
                  const __m128i idx1_1 = _mm_add_epi16(e1_1, dup_256);

            
                  const __m128i error1_1 =

            
                      _mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx1_1, 7)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 6)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 5)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 4)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 3)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 2)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 1)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_1, 0)]);


                  const __m128i idx2_1 = _mm_add_epi16(e1_1, dup_257);

            
                  const __m128i error2_1 =

            
                      _mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx2_1, 7)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 6)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 5)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 4)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 3)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 2)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 1)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_1, 0)]);


                  const __m128i error_lo_1 = _mm_unpacklo_epi16(error1_1, error2_1);

            
                  const __m128i error_hi_1 = _mm_unpackhi_epi16(error1_1, error2_1);


                  const __m128i idx1_2 = _mm_add_epi16(e1_2, dup_256);

            
                  const __m128i error1_2 =

            
                      _mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx1_2, 7)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 6)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 5)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 4)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 3)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 2)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 1)],

            
                                    error_measure_lut[_mm_extract_epi16(idx1_2, 0)]);


                  const __m128i idx2_2 = _mm_add_epi16(e1_2, dup_257);

            
                  const __m128i error2_2 =

            
                      _mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx2_2, 7)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 6)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 5)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 4)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 3)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 2)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 1)],

            
                                    error_measure_lut[_mm_extract_epi16(idx2_2, 0)]);


                  const __m128i error_lo_2 = _mm_unpacklo_epi16(error1_2, error2_2);

            
                  const __m128i error_hi_2 = _mm_unpackhi_epi16(error1_2, error2_2);


                  // Compute multipliers

            
                  const __m128i e2_inv_1 = _mm_sub_epi16(v, e2_1);

            
                  const __m128i mul_lo_1 = _mm_unpacklo_epi16(e2_inv_1, e2_1);

            
                  const __m128i mul_hi_1 = _mm_unpackhi_epi16(e2_inv_1, e2_1);


                  const __m128i e2_inv_2 = _mm_sub_epi16(v, e2_2);

            
                  const __m128i mul_lo_2 = _mm_unpacklo_epi16(e2_inv_2, e2_2);

            
                  const __m128i mul_hi_2 = _mm_unpackhi_epi16(e2_inv_2, e2_2);


                  // Multiply and accumulate

            
                  const __m128i result1_1 = _mm_madd_epi16(error_lo_1, mul_lo_1);

            
                  const __m128i result2_1 = _mm_madd_epi16(error_hi_1, mul_hi_1);

            
                  const __m128i result1_2 = _mm_madd_epi16(error_lo_2, mul_lo_2);

            
                  const __m128i result2_2 = _mm_madd_epi16(error_hi_2, mul_hi_2);


                  const __m128i partial_sum =

            
                      _mm_add_epi32(_mm_add_epi32(result1_1, result2_1),

            
                                    _mm_add_epi32(result1_2, result2_2));


                  row_error = _mm_add_epi32(row_error, partial_sum);

            
                }


                const __m128i col_error_lo = _mm_unpacklo_epi32(row_error, zero);

            
                const __m128i col_error_hi = _mm_unpackhi_epi32(row_error, zero);

            
                const __m128i col_error_temp = _mm_add_epi64(col_error_lo, col_error_hi);

            
                col_error = _mm_add_epi64(col_error, col_error_temp);

            
                // Error summation for remaining width, which is not multiple of 16

            
                if (p_width & 0xf) {

            
                  for (int l = j * 16; l < p_width; ++l) {

            
                    sum_error += (int64_t)highbd_error_measure(

            
                        dst[l + i * dst_stride] - ref[l + i * ref_stride], bd);

            
                  }

            
                }

            
              }

            
              int64_t sum_error_d_0, sum_error_d_1;

            
              xx_storel_64(&sum_error_d_0, col_error);

            
              xx_storel_64(&sum_error_d_1, _mm_srli_si128(col_error, 8));

            
              sum_error = (sum_error + sum_error_d_0 + sum_error_d_1);

            
              return sum_error;

            
            }

            
            #endif  // CONFIG_AV1_HIGHBITDEPTH


            int64_t av1_calc_frame_error_sse2(const uint8_t *const ref, int ref_stride,

            
                                              const uint8_t *const dst, int dst_stride,

            
                                              int p_width, int p_height) {

            
              int64_t sum_error = 0;

            
              int i, j;

            
              __m128i row_error, col_error;

            
              const __m128i zero = _mm_setzero_si128();

            
              const __m128i dup_256 = _mm_set1_epi16(256);

            
              col_error = zero;

            
              for (i = 0; i < (p_height); i++) {

            
                row_error = zero;

            
                for (j = 0; j < (p_width / 16); j++) {

            
                  const __m128i ref_8 =

            
                      _mm_load_si128((__m128i *)(ref + (j * 16) + (i * ref_stride)));

            
                  const __m128i dst_8 =

            
                      _mm_load_si128((__m128i *)(dst + (j * 16) + (i * dst_stride)));

            
                  const __m128i ref_16_lo = _mm_unpacklo_epi8(ref_8, zero);

            
                  const __m128i ref_16_hi = _mm_unpackhi_epi8(ref_8, zero);

            
                  const __m128i dst_16_lo = _mm_unpacklo_epi8(dst_8, zero);

            
                  const __m128i dst_16_hi = _mm_unpackhi_epi8(dst_8, zero);


                  const __m128i diff_1 =

            
                      _mm_add_epi16(_mm_sub_epi16(dst_16_lo, ref_16_lo), dup_256);

            
                  const __m128i diff_2 =

            
                      _mm_add_epi16(_mm_sub_epi16(dst_16_hi, ref_16_hi), dup_256);


                  const __m128i error_1_lo =

            
                      _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_1, 3)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_1, 2)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_1, 1)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_1, 0)]);

            
                  const __m128i error_1_hi =

            
                      _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_1, 7)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_1, 6)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_1, 5)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_1, 4)]);

            
                  const __m128i error_2_lo =

            
                      _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_2, 3)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_2, 2)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_2, 1)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_2, 0)]);

            
                  const __m128i error_2_hi =

            
                      _mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_2, 7)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_2, 6)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_2, 5)],

            
                                    error_measure_lut[_mm_extract_epi16(diff_2, 4)]);


                  const __m128i error_1 = _mm_add_epi32(error_1_lo, error_1_hi);

            
                  const __m128i error_2 = _mm_add_epi32(error_2_lo, error_2_hi);

            
                  const __m128i error_1_2 = _mm_add_epi32(error_1, error_2);


                  row_error = _mm_add_epi32(row_error, error_1_2);

            
                }

            
                const __m128i col_error_lo = _mm_unpacklo_epi32(row_error, zero);

            
                const __m128i col_error_hi = _mm_unpackhi_epi32(row_error, zero);

            
                const __m128i col_error_temp = _mm_add_epi64(col_error_lo, col_error_hi);

            
                col_error = _mm_add_epi64(col_error, col_error_temp);

            
                // Error summation for remaining width, which is not multiple of 16

            
                if (p_width & 0xf) {

            
                  for (int l = j * 16; l < p_width; ++l) {

            
                    sum_error += (int64_t)error_measure(dst[l + i * dst_stride] -

            
                                                        ref[l + i * ref_stride]);

            
                  }

            
                }

            
              }

            
              int64_t sum_error_d_0, sum_error_d_1;

            
              xx_storel_64(&sum_error_d_0, col_error);

            
              xx_storel_64(&sum_error_d_1, _mm_srli_si128(col_error, 8));

            
              sum_error = (sum_error + sum_error_d_0 + sum_error_d_1);

            
              return sum_error;

            
            }
	/*
	* Copyright (c) 2019, Alliance for Open Media. All rights reserved
	*
	* This source code is subject to the terms of the BSD 2 Clause License and
	* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
	* was not distributed with this source code in the LICENSE file, you can
	* obtain it at www.aomedia.org/license/software. If the Alliance for Open
	* Media Patent License 1.0 was not distributed with this source code in the
	* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
	*/

	#include <emmintrin.h>

	#include "aom_dsp/x86/synonyms.h"
	#include "av1/common/warped_motion.h"
	#include "av1/encoder/global_motion.h"
	#include "config/av1_rtcd.h"

	#if CONFIG_AV1_HIGHBITDEPTH
	int64_t av1_calc_highbd_frame_error_sse2(const uint16_t *const ref,
	int ref_stride,
	const uint16_t *const dst,
	int dst_stride, int p_width,
	int p_height, int bd) {
	const int b = bd - 8;
	const __m128i shift = _mm_cvtsi32_si128(b);
	const __m128i bmask = _mm_set1_epi16((1 << b) - 1);
	const __m128i v = _mm_set1_epi16(1 << b);

	int64_t sum_error = 0;
	int i, j;
	__m128i row_error, col_error;
	const __m128i zero = _mm_setzero_si128();
	const __m128i dup_256 = _mm_set1_epi16(256);
	const __m128i dup_257 = _mm_set1_epi16(257);
	col_error = zero;
	for (i = 0; i < (p_height); i++) {
	row_error = zero;
	for (j = 0; j < (p_width / 16); j++) {
	const __m128i ref_1 =
	_mm_load_si128((__m128i )(ref + (j 16) + (i * ref_stride)));
	const __m128i dst_1 =
	_mm_load_si128((__m128i )(dst + (j 16) + (i * dst_stride)));
	const __m128i ref_2 =
	_mm_load_si128((__m128i )(ref + (j 16 + 8) + (i * ref_stride)));
	const __m128i dst_2 =
	_mm_load_si128((__m128i )(dst + (j 16 + 8) + (i * dst_stride)));

	const __m128i diff_1 = _mm_sub_epi16(dst_1, ref_1);
	const __m128i diff_2 = _mm_sub_epi16(dst_2, ref_2);

	const __m128i e1_1 = _mm_sra_epi16(diff_1, shift);
	const __m128i e2_1 = _mm_and_si128(diff_1, bmask);
	const __m128i e1_2 = _mm_sra_epi16(diff_2, shift);
	const __m128i e2_2 = _mm_and_si128(diff_2, bmask);

	// For each 16-bit element in e1 and e2, we need to accumulate
	// the value:
	// error_measure_lut[256 + e1] * (v - e2) +
	// error_measure_lut[257 + e1] * e2
	// To do this, we first synthesize two 16-bit gathers, then
	// interleave the factors in such a way that we can use _mm_madd_epi16
	// to do eight 16x16->32 multiplies in one go
	const __m128i idx1_1 = _mm_add_epi16(e1_1, dup_256);
	const __m128i error1_1 =
	_mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx1_1, 7)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 6)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 5)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 4)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 3)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 2)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 1)],
	error_measure_lut[_mm_extract_epi16(idx1_1, 0)]);

	const __m128i idx2_1 = _mm_add_epi16(e1_1, dup_257);
	const __m128i error2_1 =
	_mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx2_1, 7)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 6)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 5)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 4)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 3)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 2)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 1)],
	error_measure_lut[_mm_extract_epi16(idx2_1, 0)]);

	const __m128i error_lo_1 = _mm_unpacklo_epi16(error1_1, error2_1);
	const __m128i error_hi_1 = _mm_unpackhi_epi16(error1_1, error2_1);

	const __m128i idx1_2 = _mm_add_epi16(e1_2, dup_256);
	const __m128i error1_2 =
	_mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx1_2, 7)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 6)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 5)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 4)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 3)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 2)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 1)],
	error_measure_lut[_mm_extract_epi16(idx1_2, 0)]);

	const __m128i idx2_2 = _mm_add_epi16(e1_2, dup_257);
	const __m128i error2_2 =
	_mm_set_epi16(error_measure_lut[_mm_extract_epi16(idx2_2, 7)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 6)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 5)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 4)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 3)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 2)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 1)],
	error_measure_lut[_mm_extract_epi16(idx2_2, 0)]);

	const __m128i error_lo_2 = _mm_unpacklo_epi16(error1_2, error2_2);
	const __m128i error_hi_2 = _mm_unpackhi_epi16(error1_2, error2_2);

	// Compute multipliers
	const __m128i e2_inv_1 = _mm_sub_epi16(v, e2_1);
	const __m128i mul_lo_1 = _mm_unpacklo_epi16(e2_inv_1, e2_1);
	const __m128i mul_hi_1 = _mm_unpackhi_epi16(e2_inv_1, e2_1);

	const __m128i e2_inv_2 = _mm_sub_epi16(v, e2_2);
	const __m128i mul_lo_2 = _mm_unpacklo_epi16(e2_inv_2, e2_2);
	const __m128i mul_hi_2 = _mm_unpackhi_epi16(e2_inv_2, e2_2);

	// Multiply and accumulate
	const __m128i result1_1 = _mm_madd_epi16(error_lo_1, mul_lo_1);
	const __m128i result2_1 = _mm_madd_epi16(error_hi_1, mul_hi_1);
	const __m128i result1_2 = _mm_madd_epi16(error_lo_2, mul_lo_2);
	const __m128i result2_2 = _mm_madd_epi16(error_hi_2, mul_hi_2);

	const __m128i partial_sum =
	_mm_add_epi32(_mm_add_epi32(result1_1, result2_1),
	_mm_add_epi32(result1_2, result2_2));

	row_error = _mm_add_epi32(row_error, partial_sum);
	}

	const __m128i col_error_lo = _mm_unpacklo_epi32(row_error, zero);
	const __m128i col_error_hi = _mm_unpackhi_epi32(row_error, zero);
	const __m128i col_error_temp = _mm_add_epi64(col_error_lo, col_error_hi);
	col_error = _mm_add_epi64(col_error, col_error_temp);
	// Error summation for remaining width, which is not multiple of 16
	if (p_width & 0xf) {
	for (int l = j * 16; l < p_width; ++l) {
	sum_error += (int64_t)highbd_error_measure(
	dst[l + i * dst_stride] - ref[l + i * ref_stride], bd);
	}
	}
	}
	int64_t sum_error_d_0, sum_error_d_1;
	xx_storel_64(&sum_error_d_0, col_error);
	xx_storel_64(&sum_error_d_1, _mm_srli_si128(col_error, 8));
	sum_error = (sum_error + sum_error_d_0 + sum_error_d_1);
	return sum_error;
	}
	#endif // CONFIG_AV1_HIGHBITDEPTH

	int64_t av1_calc_frame_error_sse2(const uint8_t *const ref, int ref_stride,
	const uint8_t *const dst, int dst_stride,
	int p_width, int p_height) {
	int64_t sum_error = 0;
	int i, j;
	__m128i row_error, col_error;
	const __m128i zero = _mm_setzero_si128();
	const __m128i dup_256 = _mm_set1_epi16(256);
	col_error = zero;
	for (i = 0; i < (p_height); i++) {
	row_error = zero;
	for (j = 0; j < (p_width / 16); j++) {
	const __m128i ref_8 =
	_mm_load_si128((__m128i )(ref + (j 16) + (i * ref_stride)));
	const __m128i dst_8 =
	_mm_load_si128((__m128i )(dst + (j 16) + (i * dst_stride)));
	const __m128i ref_16_lo = _mm_unpacklo_epi8(ref_8, zero);
	const __m128i ref_16_hi = _mm_unpackhi_epi8(ref_8, zero);
	const __m128i dst_16_lo = _mm_unpacklo_epi8(dst_8, zero);
	const __m128i dst_16_hi = _mm_unpackhi_epi8(dst_8, zero);

	const __m128i diff_1 =
	_mm_add_epi16(_mm_sub_epi16(dst_16_lo, ref_16_lo), dup_256);
	const __m128i diff_2 =
	_mm_add_epi16(_mm_sub_epi16(dst_16_hi, ref_16_hi), dup_256);

	const __m128i error_1_lo =
	_mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_1, 3)],
	error_measure_lut[_mm_extract_epi16(diff_1, 2)],
	error_measure_lut[_mm_extract_epi16(diff_1, 1)],
	error_measure_lut[_mm_extract_epi16(diff_1, 0)]);
	const __m128i error_1_hi =
	_mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_1, 7)],
	error_measure_lut[_mm_extract_epi16(diff_1, 6)],
	error_measure_lut[_mm_extract_epi16(diff_1, 5)],
	error_measure_lut[_mm_extract_epi16(diff_1, 4)]);
	const __m128i error_2_lo =
	_mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_2, 3)],
	error_measure_lut[_mm_extract_epi16(diff_2, 2)],
	error_measure_lut[_mm_extract_epi16(diff_2, 1)],
	error_measure_lut[_mm_extract_epi16(diff_2, 0)]);
	const __m128i error_2_hi =
	_mm_set_epi32(error_measure_lut[_mm_extract_epi16(diff_2, 7)],
	error_measure_lut[_mm_extract_epi16(diff_2, 6)],
	error_measure_lut[_mm_extract_epi16(diff_2, 5)],
	error_measure_lut[_mm_extract_epi16(diff_2, 4)]);

	const __m128i error_1 = _mm_add_epi32(error_1_lo, error_1_hi);
	const __m128i error_2 = _mm_add_epi32(error_2_lo, error_2_hi);
	const __m128i error_1_2 = _mm_add_epi32(error_1, error_2);

	row_error = _mm_add_epi32(row_error, error_1_2);
	}
	const __m128i col_error_lo = _mm_unpacklo_epi32(row_error, zero);
	const __m128i col_error_hi = _mm_unpackhi_epi32(row_error, zero);
	const __m128i col_error_temp = _mm_add_epi64(col_error_lo, col_error_hi);
	col_error = _mm_add_epi64(col_error, col_error_temp);
	// Error summation for remaining width, which is not multiple of 16
	if (p_width & 0xf) {
	for (int l = j * 16; l < p_width; ++l) {
	sum_error += (int64_t)error_measure(dst[l + i * dst_stride] -
	ref[l + i * ref_stride]);
	}
	}
	}
	int64_t sum_error_d_0, sum_error_d_1;
	xx_storel_64(&sum_error_d_0, col_error);
	xx_storel_64(&sum_error_d_1, _mm_srli_si128(col_error, 8));
	sum_error = (sum_error + sum_error_d_0 + sum_error_d_1);
	return sum_error;
	}