av1/common/x86/cfl_ssse3.c - avm - Git at Google

 /*
  * Copyright (c) 2021, Alliance for Open Media. All rights reserved
  *
  * This source code is subject to the terms of the BSD 3-Clause Clear License
  * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
  * License was not distributed with this source code in the LICENSE file, you
  * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/.  If the
  * Alliance for Open Media Patent License 1.0 was not distributed with this
  * source code in the PATENTS file, you can obtain it at
  * aomedia.org/license/patent-license/.
  */

 #include <tmmintrin.h>

 #include "config/av1_rtcd.h"

 #include "av1/common/cfl.h"

 #include "av1/common/x86/cfl_simd.h"

 // Store 32-bit integer from the first element of a into memory.
 static INLINE void _mm_storeh_epi32(__m128i const *mem_addr, __m128i a) {
   *((int *)mem_addr) = _mm_cvtsi128_si32(a);
 }

 /**
  * Adds 4 pixels (in a 2x2 grid) and multiplies them by 2. Resulting in a more
  * precise version of a box filter 4:2:0 pixel subsampling in Q3.
  *
  * The CfL prediction buffer is always of size CFL_BUF_SQUARE. However, the
  * active area is specified using width and height.
  *
  * Note: We don't need to worry about going over the active area, as long as we
  * stay inside the CfL prediction buffer.
  */
 static INLINE void cfl_luma_subsampling_420_hbd_ssse3(const uint16_t *input,
                                                       int input_stride,
                                                       uint16_t *pred_buf_q3,
                                                       int width, int height) {
   const uint16_t *end = pred_buf_q3 + (height >> 1) * CFL_BUF_LINE;
   const int luma_stride = input_stride << 1;
   do {
     if (width == 4) {
       const __m128i top = _mm_loadl_epi64((__m128i *)input);
       const __m128i bot = _mm_loadl_epi64((__m128i *)(input + input_stride));
       __m128i sum = _mm_add_epi16(top, bot);
       sum = _mm_hadd_epi16(sum, sum);
       *((int *)pred_buf_q3) = _mm_cvtsi128_si32(_mm_add_epi16(sum, sum));
     } else {
       const __m128i top = _mm_loadu_si128((__m128i *)input);
       const __m128i bot = _mm_loadu_si128((__m128i *)(input + input_stride));
       __m128i sum = _mm_add_epi16(top, bot);
       if (width == 8) {
         sum = _mm_hadd_epi16(sum, sum);
         _mm_storel_epi64((__m128i *)pred_buf_q3, _mm_add_epi16(sum, sum));
       } else {
         const __m128i top_1 = _mm_loadu_si128(((__m128i *)input) + 1);
         const __m128i bot_1 =
             _mm_loadu_si128(((__m128i *)(input + input_stride)) + 1);
         sum = _mm_hadd_epi16(sum, _mm_add_epi16(top_1, bot_1));
         _mm_storeu_si128((__m128i *)pred_buf_q3, _mm_add_epi16(sum, sum));
         if (width == 32) {
           const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
           const __m128i bot_2 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 2);
           const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
           const __m128i bot_3 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 3);
           const __m128i sum_2 = _mm_add_epi16(top_2, bot_2);
           const __m128i sum_3 = _mm_add_epi16(top_3, bot_3);
           __m128i next_sum = _mm_hadd_epi16(sum_2, sum_3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 1,
                            _mm_add_epi16(next_sum, next_sum));
         } else if (width == 64) {
           const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
           const __m128i bot_2 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 2);
           const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
           const __m128i bot_3 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 3);
           const __m128i sum_2 = _mm_add_epi16(top_2, bot_2);
           const __m128i sum_3 = _mm_add_epi16(top_3, bot_3);
           __m128i next_sum = _mm_hadd_epi16(sum_2, sum_3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 1,
                            _mm_add_epi16(next_sum, next_sum));

           const __m128i top_4 = _mm_loadu_si128(((__m128i *)input) + 4);
           const __m128i bot_4 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 4);
           const __m128i top_5 = _mm_loadu_si128(((__m128i *)input) + 5);
           const __m128i bot_5 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 5);
           const __m128i sum_4 = _mm_add_epi16(top_4, bot_4);
           const __m128i sum_5 = _mm_add_epi16(top_5, bot_5);
           __m128i next_sum_2 = _mm_hadd_epi16(sum_4, sum_5);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 2,
                            _mm_add_epi16(next_sum_2, next_sum_2));

           const __m128i top_6 = _mm_loadu_si128(((__m128i *)input) + 6);
           const __m128i bot_6 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 6);
           const __m128i top_7 = _mm_loadu_si128(((__m128i *)input) + 7);
           const __m128i bot_7 =
               _mm_loadu_si128(((__m128i *)(input + input_stride)) + 7);
           const __m128i sum_6 = _mm_add_epi16(top_6, bot_6);
           const __m128i sum_7 = _mm_add_epi16(top_7, bot_7);
           __m128i next_sum_3 = _mm_hadd_epi16(sum_6, sum_7);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 3,
                            _mm_add_epi16(next_sum_3, next_sum_3));
         }
       }
     }
     input += luma_stride;
   } while ((pred_buf_q3 += CFL_BUF_LINE) < end);
 }

 /**
  * Adds 2 pixels (in a 2x1 grid) and multiplies them by 4. Resulting in a more
  * precise version of a box filter 4:2:2 pixel subsampling in Q3.
  *
  * The CfL prediction buffer is always of size CFL_BUF_SQUARE. However, the
  * active area is specified using width and height.
  *
  * Note: We don't need to worry about going over the active area, as long as we
  * stay inside the CfL prediction buffer.
  */
 static INLINE void cfl_luma_subsampling_422_hbd_ssse3(const uint16_t *input,
                                                       int input_stride,
                                                       uint16_t *pred_buf_q3,
                                                       int width, int height) {
   __m128i *pred_buf_m128i = (__m128i *)pred_buf_q3;
   const __m128i *end = pred_buf_m128i + height * CFL_BUF_LINE_I128;
   do {
     if (width == 4) {
       const __m128i top = _mm_loadl_epi64((__m128i *)input);
       const __m128i sum = _mm_slli_epi16(_mm_hadd_epi16(top, top), 2);
       _mm_storeh_epi32(pred_buf_m128i, sum);
     } else {
       const __m128i top = _mm_loadu_si128((__m128i *)input);
       if (width == 8) {
         const __m128i sum = _mm_slli_epi16(_mm_hadd_epi16(top, top), 2);
         _mm_storel_epi64(pred_buf_m128i, sum);
       } else {
         const __m128i top_1 = _mm_loadu_si128(((__m128i *)input) + 1);
         const __m128i sum = _mm_slli_epi16(_mm_hadd_epi16(top, top_1), 2);
         _mm_storeu_si128(pred_buf_m128i, sum);
         if (width == 32) {
           const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
           const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
           const __m128i sum_1 = _mm_slli_epi16(_mm_hadd_epi16(top_2, top_3), 2);
           _mm_storeu_si128(pred_buf_m128i + 1, sum_1);
         } else if (width == 64) {
           const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
           const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
           const __m128i sum_1 = _mm_slli_epi16(_mm_hadd_epi16(top_2, top_3), 2);
           _mm_storeu_si128(pred_buf_m128i + 1, sum_1);
           const __m128i top_4 = _mm_loadu_si128(((__m128i *)input) + 4);
           const __m128i top_5 = _mm_loadu_si128(((__m128i *)input) + 5);
           const __m128i sum_2 = _mm_slli_epi16(_mm_hadd_epi16(top_4, top_5), 2);
           _mm_storeu_si128(pred_buf_m128i + 2, sum_2);
           const __m128i top_6 = _mm_loadu_si128(((__m128i *)input) + 6);
           const __m128i top_7 = _mm_loadu_si128(((__m128i *)input) + 7);
           const __m128i sum_3 = _mm_slli_epi16(_mm_hadd_epi16(top_6, top_7), 2);
           _mm_storeu_si128(pred_buf_m128i + 3, sum_3);
         }
       }
     }
     pred_buf_m128i += CFL_BUF_LINE_I128;
     input += input_stride;
   } while (pred_buf_m128i < end);
 }

 static INLINE void cfl_luma_subsampling_444_hbd_ssse3(const uint16_t *input,
                                                       int input_stride,
                                                       uint16_t *pred_buf_q3,
                                                       int width, int height) {
   const uint16_t *end = pred_buf_q3 + height * CFL_BUF_LINE;
   do {
     if (width == 4) {
       const __m128i row = _mm_slli_epi16(_mm_loadl_epi64((__m128i *)input), 3);
       _mm_storel_epi64((__m128i *)pred_buf_q3, row);
     } else {
       const __m128i row = _mm_slli_epi16(_mm_loadu_si128((__m128i *)input), 3);
       _mm_storeu_si128((__m128i *)pred_buf_q3, row);
       if (width >= 16) {
         __m128i row_1 = _mm_loadu_si128(((__m128i *)input) + 1);
         row_1 = _mm_slli_epi16(row_1, 3);
         _mm_storeu_si128(((__m128i *)pred_buf_q3) + 1, row_1);
         if (width == 32) {
           __m128i row_2 = _mm_loadu_si128(((__m128i *)input) + 2);
           row_2 = _mm_slli_epi16(row_2, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 2, row_2);
           __m128i row_3 = _mm_loadu_si128(((__m128i *)input) + 3);
           row_3 = _mm_slli_epi16(row_3, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 3, row_3);
         } else if (width == 64) {
           __m128i row_2 = _mm_loadu_si128(((__m128i *)input) + 2);
           row_2 = _mm_slli_epi16(row_2, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 2, row_2);
           __m128i row_3 = _mm_loadu_si128(((__m128i *)input) + 3);
           row_3 = _mm_slli_epi16(row_3, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 3, row_3);
           __m128i row_4 = _mm_loadu_si128(((__m128i *)input) + 4);
           row_4 = _mm_slli_epi16(row_4, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 4, row_4);
           __m128i row_5 = _mm_loadu_si128(((__m128i *)input) + 5);
           row_5 = _mm_slli_epi16(row_5, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 5, row_5);
           __m128i row_6 = _mm_loadu_si128(((__m128i *)input) + 6);
           row_6 = _mm_slli_epi16(row_6, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 6, row_6);
           __m128i row_7 = _mm_loadu_si128(((__m128i *)input) + 7);
           row_7 = _mm_slli_epi16(row_7, 3);
           _mm_storeu_si128(((__m128i *)pred_buf_q3) + 7, row_7);
         }
       }
     }
     input += input_stride;
     pred_buf_q3 += CFL_BUF_LINE;
   } while (pred_buf_q3 < end);
 }

 CFL_GET_SUBSAMPLE_FUNCTION(ssse3)

 static INLINE __m128i predict_unclipped(const __m128i *input, __m128i alpha_q12,
                                         __m128i alpha_sign, __m128i dc_q0) {
   __m128i ac_q3 = _mm_loadu_si128(input);
   __m128i ac_sign = _mm_sign_epi16(alpha_sign, ac_q3);
   __m128i scaled_luma_q0 = _mm_mulhrs_epi16(_mm_abs_epi16(ac_q3), alpha_q12);
   scaled_luma_q0 = _mm_sign_epi16(scaled_luma_q0, ac_sign);
   return _mm_add_epi16(scaled_luma_q0, dc_q0);
 }

 static INLINE __m128i highbd_max_epi16(int bd) {
   const __m128i neg_one = _mm_set1_epi16(-1);
   // (1 << bd) - 1 => -(-1 << bd) -1 => -1 - (-1 << bd) => -1 ^ (-1 << bd)
   return _mm_xor_si128(_mm_slli_epi16(neg_one, bd), neg_one);
 }

 static INLINE __m128i highbd_clamp_epi16(__m128i u, __m128i zero, __m128i max) {
   return _mm_max_epi16(_mm_min_epi16(u, max), zero);
 }

 static INLINE void cfl_predict_hbd_ssse3(const int16_t *pred_buf_q3,
                                          uint16_t *dst, int dst_stride,
                                          int alpha_q3, int bd, int width,
                                          int height) {
   const __m128i alpha_sign = _mm_set1_epi16(alpha_q3);
   const __m128i alpha_q12 =
       _mm_slli_epi16(_mm_abs_epi16(alpha_sign), (9 - CFL_ADD_BITS_ALPHA));
   const __m128i dc_q0 = _mm_set1_epi16(*dst);
   const __m128i max = highbd_max_epi16(bd);
   const __m128i zeros = _mm_setzero_si128();
   __m128i *row = (__m128i *)pred_buf_q3;
   const __m128i *row_end = row + height * CFL_BUF_LINE_I128;
   do {
     __m128i res = predict_unclipped(row, alpha_q12, alpha_sign, dc_q0);
     res = highbd_clamp_epi16(res, zeros, max);
     if (width == 4) {
       _mm_storel_epi64((__m128i *)dst, res);
     } else {
       _mm_storeu_si128((__m128i *)dst, res);
     }
     if (width >= 16) {
       const __m128i res_1 =
           predict_unclipped(row + 1, alpha_q12, alpha_sign, dc_q0);
       _mm_storeu_si128(((__m128i *)dst) + 1,
                        highbd_clamp_epi16(res_1, zeros, max));
     }
     if (width == 32) {
       const __m128i res_2 =
           predict_unclipped(row + 2, alpha_q12, alpha_sign, dc_q0);
       _mm_storeu_si128((__m128i *)(dst + 16),
                        highbd_clamp_epi16(res_2, zeros, max));
       const __m128i res_3 =
           predict_unclipped(row + 3, alpha_q12, alpha_sign, dc_q0);
       _mm_storeu_si128((__m128i *)(dst + 24),
                        highbd_clamp_epi16(res_3, zeros, max));
     }
     dst += dst_stride;
   } while ((row += CFL_BUF_LINE_I128) < row_end);
 }

 CFL_PREDICT_FN(ssse3, hbd)
	/*
	* Copyright (c) 2021, Alliance for Open Media. All rights reserved
	*
	* This source code is subject to the terms of the BSD 3-Clause Clear License
	* and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
	* License was not distributed with this source code in the LICENSE file, you
	* can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the
	* Alliance for Open Media Patent License 1.0 was not distributed with this
	* source code in the PATENTS file, you can obtain it at
	* aomedia.org/license/patent-license/.
	*/

	#include <tmmintrin.h>

	#include "config/av1_rtcd.h"

	#include "av1/common/cfl.h"

	#include "av1/common/x86/cfl_simd.h"

	// Store 32-bit integer from the first element of a into memory.
	static INLINE void _mm_storeh_epi32(__m128i const *mem_addr, __m128i a) {
	((int )mem_addr) = _mm_cvtsi128_si32(a);
	}

	/**
	* Adds 4 pixels (in a 2x2 grid) and multiplies them by 2. Resulting in a more
	* precise version of a box filter 4:2:0 pixel subsampling in Q3.
	*
	* The CfL prediction buffer is always of size CFL_BUF_SQUARE. However, the
	* active area is specified using width and height.
	*
	* Note: We don't need to worry about going over the active area, as long as we
	* stay inside the CfL prediction buffer.
	*/
	static INLINE void cfl_luma_subsampling_420_hbd_ssse3(const uint16_t *input,
	int input_stride,
	uint16_t *pred_buf_q3,
	int width, int height) {
	const uint16_t end = pred_buf_q3 + (height >> 1) CFL_BUF_LINE;
	const int luma_stride = input_stride << 1;
	do {
	if (width == 4) {
	const __m128i top = _mm_loadl_epi64((__m128i *)input);
	const __m128i bot = _mm_loadl_epi64((__m128i *)(input + input_stride));
	__m128i sum = _mm_add_epi16(top, bot);
	sum = _mm_hadd_epi16(sum, sum);
	((int )pred_buf_q3) = _mm_cvtsi128_si32(_mm_add_epi16(sum, sum));
	} else {
	const __m128i top = _mm_loadu_si128((__m128i *)input);
	const __m128i bot = _mm_loadu_si128((__m128i *)(input + input_stride));
	__m128i sum = _mm_add_epi16(top, bot);
	if (width == 8) {
	sum = _mm_hadd_epi16(sum, sum);
	_mm_storel_epi64((__m128i *)pred_buf_q3, _mm_add_epi16(sum, sum));
	} else {
	const __m128i top_1 = _mm_loadu_si128(((__m128i *)input) + 1);
	const __m128i bot_1 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 1);
	sum = _mm_hadd_epi16(sum, _mm_add_epi16(top_1, bot_1));
	_mm_storeu_si128((__m128i *)pred_buf_q3, _mm_add_epi16(sum, sum));
	if (width == 32) {
	const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
	const __m128i bot_2 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 2);
	const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
	const __m128i bot_3 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 3);
	const __m128i sum_2 = _mm_add_epi16(top_2, bot_2);
	const __m128i sum_3 = _mm_add_epi16(top_3, bot_3);
	__m128i next_sum = _mm_hadd_epi16(sum_2, sum_3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 1,
	_mm_add_epi16(next_sum, next_sum));
	} else if (width == 64) {
	const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
	const __m128i bot_2 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 2);
	const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
	const __m128i bot_3 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 3);
	const __m128i sum_2 = _mm_add_epi16(top_2, bot_2);
	const __m128i sum_3 = _mm_add_epi16(top_3, bot_3);
	__m128i next_sum = _mm_hadd_epi16(sum_2, sum_3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 1,
	_mm_add_epi16(next_sum, next_sum));

	const __m128i top_4 = _mm_loadu_si128(((__m128i *)input) + 4);
	const __m128i bot_4 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 4);
	const __m128i top_5 = _mm_loadu_si128(((__m128i *)input) + 5);
	const __m128i bot_5 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 5);
	const __m128i sum_4 = _mm_add_epi16(top_4, bot_4);
	const __m128i sum_5 = _mm_add_epi16(top_5, bot_5);
	__m128i next_sum_2 = _mm_hadd_epi16(sum_4, sum_5);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 2,
	_mm_add_epi16(next_sum_2, next_sum_2));

	const __m128i top_6 = _mm_loadu_si128(((__m128i *)input) + 6);
	const __m128i bot_6 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 6);
	const __m128i top_7 = _mm_loadu_si128(((__m128i *)input) + 7);
	const __m128i bot_7 =
	_mm_loadu_si128(((__m128i *)(input + input_stride)) + 7);
	const __m128i sum_6 = _mm_add_epi16(top_6, bot_6);
	const __m128i sum_7 = _mm_add_epi16(top_7, bot_7);
	__m128i next_sum_3 = _mm_hadd_epi16(sum_6, sum_7);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 3,
	_mm_add_epi16(next_sum_3, next_sum_3));
	}
	}
	}
	input += luma_stride;
	} while ((pred_buf_q3 += CFL_BUF_LINE) < end);
	}

	/**
	* Adds 2 pixels (in a 2x1 grid) and multiplies them by 4. Resulting in a more
	* precise version of a box filter 4:2:2 pixel subsampling in Q3.
	*
	* The CfL prediction buffer is always of size CFL_BUF_SQUARE. However, the
	* active area is specified using width and height.
	*
	* Note: We don't need to worry about going over the active area, as long as we
	* stay inside the CfL prediction buffer.
	*/
	static INLINE void cfl_luma_subsampling_422_hbd_ssse3(const uint16_t *input,
	int input_stride,
	uint16_t *pred_buf_q3,
	int width, int height) {
	__m128i pred_buf_m128i = (__m128i )pred_buf_q3;
	const __m128i end = pred_buf_m128i + height CFL_BUF_LINE_I128;
	do {
	if (width == 4) {
	const __m128i top = _mm_loadl_epi64((__m128i *)input);
	const __m128i sum = _mm_slli_epi16(_mm_hadd_epi16(top, top), 2);
	_mm_storeh_epi32(pred_buf_m128i, sum);
	} else {
	const __m128i top = _mm_loadu_si128((__m128i *)input);
	if (width == 8) {
	const __m128i sum = _mm_slli_epi16(_mm_hadd_epi16(top, top), 2);
	_mm_storel_epi64(pred_buf_m128i, sum);
	} else {
	const __m128i top_1 = _mm_loadu_si128(((__m128i *)input) + 1);
	const __m128i sum = _mm_slli_epi16(_mm_hadd_epi16(top, top_1), 2);
	_mm_storeu_si128(pred_buf_m128i, sum);
	if (width == 32) {
	const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
	const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
	const __m128i sum_1 = _mm_slli_epi16(_mm_hadd_epi16(top_2, top_3), 2);
	_mm_storeu_si128(pred_buf_m128i + 1, sum_1);
	} else if (width == 64) {
	const __m128i top_2 = _mm_loadu_si128(((__m128i *)input) + 2);
	const __m128i top_3 = _mm_loadu_si128(((__m128i *)input) + 3);
	const __m128i sum_1 = _mm_slli_epi16(_mm_hadd_epi16(top_2, top_3), 2);
	_mm_storeu_si128(pred_buf_m128i + 1, sum_1);
	const __m128i top_4 = _mm_loadu_si128(((__m128i *)input) + 4);
	const __m128i top_5 = _mm_loadu_si128(((__m128i *)input) + 5);
	const __m128i sum_2 = _mm_slli_epi16(_mm_hadd_epi16(top_4, top_5), 2);
	_mm_storeu_si128(pred_buf_m128i + 2, sum_2);
	const __m128i top_6 = _mm_loadu_si128(((__m128i *)input) + 6);
	const __m128i top_7 = _mm_loadu_si128(((__m128i *)input) + 7);
	const __m128i sum_3 = _mm_slli_epi16(_mm_hadd_epi16(top_6, top_7), 2);
	_mm_storeu_si128(pred_buf_m128i + 3, sum_3);
	}
	}
	}
	pred_buf_m128i += CFL_BUF_LINE_I128;
	input += input_stride;
	} while (pred_buf_m128i < end);
	}

	static INLINE void cfl_luma_subsampling_444_hbd_ssse3(const uint16_t *input,
	int input_stride,
	uint16_t *pred_buf_q3,
	int width, int height) {
	const uint16_t end = pred_buf_q3 + height CFL_BUF_LINE;
	do {
	if (width == 4) {
	const __m128i row = _mm_slli_epi16(_mm_loadl_epi64((__m128i *)input), 3);
	_mm_storel_epi64((__m128i *)pred_buf_q3, row);
	} else {
	const __m128i row = _mm_slli_epi16(_mm_loadu_si128((__m128i *)input), 3);
	_mm_storeu_si128((__m128i *)pred_buf_q3, row);
	if (width >= 16) {
	__m128i row_1 = _mm_loadu_si128(((__m128i *)input) + 1);
	row_1 = _mm_slli_epi16(row_1, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 1, row_1);
	if (width == 32) {
	__m128i row_2 = _mm_loadu_si128(((__m128i *)input) + 2);
	row_2 = _mm_slli_epi16(row_2, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 2, row_2);
	__m128i row_3 = _mm_loadu_si128(((__m128i *)input) + 3);
	row_3 = _mm_slli_epi16(row_3, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 3, row_3);
	} else if (width == 64) {
	__m128i row_2 = _mm_loadu_si128(((__m128i *)input) + 2);
	row_2 = _mm_slli_epi16(row_2, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 2, row_2);
	__m128i row_3 = _mm_loadu_si128(((__m128i *)input) + 3);
	row_3 = _mm_slli_epi16(row_3, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 3, row_3);
	__m128i row_4 = _mm_loadu_si128(((__m128i *)input) + 4);
	row_4 = _mm_slli_epi16(row_4, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 4, row_4);
	__m128i row_5 = _mm_loadu_si128(((__m128i *)input) + 5);
	row_5 = _mm_slli_epi16(row_5, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 5, row_5);
	__m128i row_6 = _mm_loadu_si128(((__m128i *)input) + 6);
	row_6 = _mm_slli_epi16(row_6, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 6, row_6);
	__m128i row_7 = _mm_loadu_si128(((__m128i *)input) + 7);
	row_7 = _mm_slli_epi16(row_7, 3);
	_mm_storeu_si128(((__m128i *)pred_buf_q3) + 7, row_7);
	}
	}
	}
	input += input_stride;
	pred_buf_q3 += CFL_BUF_LINE;
	} while (pred_buf_q3 < end);
	}

	CFL_GET_SUBSAMPLE_FUNCTION(ssse3)

	static INLINE __m128i predict_unclipped(const __m128i *input, __m128i alpha_q12,
	__m128i alpha_sign, __m128i dc_q0) {
	__m128i ac_q3 = _mm_loadu_si128(input);
	__m128i ac_sign = _mm_sign_epi16(alpha_sign, ac_q3);
	__m128i scaled_luma_q0 = _mm_mulhrs_epi16(_mm_abs_epi16(ac_q3), alpha_q12);
	scaled_luma_q0 = _mm_sign_epi16(scaled_luma_q0, ac_sign);
	return _mm_add_epi16(scaled_luma_q0, dc_q0);
	}

	static INLINE __m128i highbd_max_epi16(int bd) {
	const __m128i neg_one = _mm_set1_epi16(-1);
	// (1 << bd) - 1 => -(-1 << bd) -1 => -1 - (-1 << bd) => -1 ^ (-1 << bd)
	return _mm_xor_si128(_mm_slli_epi16(neg_one, bd), neg_one);
	}

	static INLINE __m128i highbd_clamp_epi16(__m128i u, __m128i zero, __m128i max) {
	return _mm_max_epi16(_mm_min_epi16(u, max), zero);
	}

	static INLINE void cfl_predict_hbd_ssse3(const int16_t *pred_buf_q3,
	uint16_t *dst, int dst_stride,
	int alpha_q3, int bd, int width,
	int height) {
	const __m128i alpha_sign = _mm_set1_epi16(alpha_q3);
	const __m128i alpha_q12 =
	_mm_slli_epi16(_mm_abs_epi16(alpha_sign), (9 - CFL_ADD_BITS_ALPHA));
	const __m128i dc_q0 = _mm_set1_epi16(*dst);
	const __m128i max = highbd_max_epi16(bd);
	const __m128i zeros = _mm_setzero_si128();
	__m128i row = (__m128i )pred_buf_q3;
	const __m128i row_end = row + height CFL_BUF_LINE_I128;
	do {
	__m128i res = predict_unclipped(row, alpha_q12, alpha_sign, dc_q0);
	res = highbd_clamp_epi16(res, zeros, max);
	if (width == 4) {
	_mm_storel_epi64((__m128i *)dst, res);
	} else {
	_mm_storeu_si128((__m128i *)dst, res);
	}
	if (width >= 16) {
	const __m128i res_1 =
	predict_unclipped(row + 1, alpha_q12, alpha_sign, dc_q0);
	_mm_storeu_si128(((__m128i *)dst) + 1,
	highbd_clamp_epi16(res_1, zeros, max));
	}
	if (width == 32) {
	const __m128i res_2 =
	predict_unclipped(row + 2, alpha_q12, alpha_sign, dc_q0);
	_mm_storeu_si128((__m128i *)(dst + 16),
	highbd_clamp_epi16(res_2, zeros, max));
	const __m128i res_3 =
	predict_unclipped(row + 3, alpha_q12, alpha_sign, dc_q0);
	_mm_storeu_si128((__m128i *)(dst + 24),
	highbd_clamp_epi16(res_3, zeros, max));
	}
	dst += dst_stride;
	} while ((row += CFL_BUF_LINE_I128) < row_end);
	}

	CFL_PREDICT_FN(ssse3, hbd)