av1/common/convolve_2d_sr_hwy.h

/*
 * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#ifndef AOM_AV1_COMMON_CONVOLVE_2D_SR_HWY_H_
#define AOM_AV1_COMMON_CONVOLVE_2D_SR_HWY_H_

#include "av1/common/convolve.h"
#include "av1/common/filter.h"
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
#include "third_party/highway/hwy/highway.h"

HWY_BEFORE_NAMESPACE();

namespace {
namespace HWY_NAMESPACE {

namespace hn = hwy::HWY_NAMESPACE;

constexpr hn::ScalableTag<uint8_t> uint8xN_tag;
constexpr hn::ScalableTag<int8_t> int8xN_tag;
constexpr hn::ScalableTag<int16_t> int16xN_tag;
constexpr hn::ScalableTag<uint32_t> uint32xN_tag;
constexpr hn::ScalableTag<int32_t> int32xN_tag;

constexpr hn::CappedTag<uint8_t, 16> uint8x16_capped_tag;
constexpr hn::CappedTag<uint8_t, 8> uint8x8_capped_tag;

using UVec8 = hn::Vec<decltype(uint8xN_tag)>;
using IVec8 = hn::Vec<decltype(int8xN_tag)>;
using IVec16 = hn::Vec<decltype(int16xN_tag)>;
using UVec32 = hn::Vec<decltype(uint32xN_tag)>;
using IVec32 = hn::Vec<decltype(int32xN_tag)>;

HWY_ALIGN constexpr uint8_t kFilt1[16] = { 0, 1, 1, 2, 2, 3, 3, 4,
                                           4, 5, 5, 6, 6, 7, 7, 8 };
HWY_ALIGN constexpr uint8_t kFilt2[16] = { 2, 3, 3, 4, 4, 5, 5, 6,
                                           6, 7, 7, 8, 8, 9, 9, 10 };
HWY_ALIGN constexpr uint8_t kFilt3[16] = { 4, 5, 5, 6,  6,  7,  7,  8,
                                           8, 9, 9, 10, 10, 11, 11, 12 };
HWY_ALIGN constexpr uint8_t kFilt4[16] = { 6,  7,  7,  8,  8,  9,  9,  10,
                                           10, 11, 11, 12, 12, 13, 13, 14 };

// Horizontal convolve helpers
template <int taps, class D16, class V8, class VI8>
HWY_ATTR HWY_INLINE hn::VFromD<D16> ConvolveLowbdX(D16 d16, V8 data,
                                                   const V8 *masks,
                                                   const VI8 *coeffs) {
  constexpr int num_coeffs = taps / 2;
  hn::VFromD<D16> prods[4];
  for (int k = 0; k < num_coeffs; ++k) {
    auto s = hn::TableLookupBytes(data, masks[k]);
    prods[k] = hn::SatWidenMulPairwiseAdd(d16, s, coeffs[k]);
  }

  HWY_IF_CONSTEXPR(num_coeffs == 1) { return prods[0]; }
  HWY_IF_CONSTEXPR(num_coeffs == 2) { return hn::Add(prods[0], prods[1]); }
  HWY_IF_CONSTEXPR(num_coeffs == 3) {
    return hn::Add(hn::Add(prods[0], prods[1]), prods[2]);
  }
  HWY_IF_CONSTEXPR(num_coeffs == 4) {
    return hn::Add(hn::Add(prods[0], prods[1]), hn::Add(prods[2], prods[3]));
  }
}

template <int taps, class D16, class V8, class VI8>
HWY_ATTR HWY_INLINE hn::VFromD<D16> ConvolveLowbdXShuffleFree(
    D16 d16, V8 data, const VI8 *coeffs) {
  // We only support AVX2 and AVX512, which have > 8 lanes of int16_t.
  static_assert(hn::MaxLanes(D16()) > 8, "Only AVX2 and AVX512 are supported");
  constexpr int num_coeffs = taps / 2;

  using D8_twice = hn::DFromV<V8>;
  const D8_twice d8_twice;

  hn::VFromD<D16> prods[4];
  auto mask1 = hn::LoadDup128(d8_twice, kFilt1);
  prods[0] = hn::SatWidenMulPairwiseAdd(d16, hn::TableLookupBytes(data, mask1),
                                        coeffs[0]);

  HWY_IF_CONSTEXPR(num_coeffs >= 2) {
    auto mask2 = hn::LoadDup128(d8_twice, kFilt2);
    prods[1] = hn::SatWidenMulPairwiseAdd(
        d16, hn::TableLookupBytes(data, mask2), coeffs[1]);
  }
  HWY_IF_CONSTEXPR(num_coeffs >= 3) {
    auto mask3 = hn::LoadDup128(d8_twice, kFilt3);
    prods[2] = hn::SatWidenMulPairwiseAdd(
        d16, hn::TableLookupBytes(data, mask3), coeffs[2]);
  }
  HWY_IF_CONSTEXPR(num_coeffs >= 4) {
    auto mask4 = hn::LoadDup128(d8_twice, kFilt4);
    prods[3] = hn::SatWidenMulPairwiseAdd(
        d16, hn::TableLookupBytes(data, mask4), coeffs[3]);
  }

  HWY_IF_CONSTEXPR(num_coeffs == 1) { return prods[0]; }
  HWY_IF_CONSTEXPR(num_coeffs == 2) { return hn::Add(prods[0], prods[1]); }
  HWY_IF_CONSTEXPR(num_coeffs == 3) {
    return hn::Add(hn::Add(prods[0], prods[1]), prods[2]);
  }
  HWY_IF_CONSTEXPR(num_coeffs == 4) {
    return hn::Add(hn::Add(prods[0], prods[1]), hn::Add(prods[2], prods[3]));
  }
}

template <class D16, class V8, class V16_coeff>
HWY_ATTR HWY_INLINE hn::VFromD<D16> ConvolveLowbdX12TapFromS(
    D16 d16, V8 s0, V8 s1, V8 s2, V8 s3, V8 s4, V8 s5, V16_coeff coeff0,
    V16_coeff coeff1, V16_coeff coeff2, V16_coeff coeff3, V16_coeff coeff4,
    V16_coeff coeff5) {
  using D16_half = hn::Half<D16>;
  using D32 = hn::Rebind<int32_t, D16_half>;
  const D32 d32;
  const D16_half d16_half;

  auto s0_low_16 = hn::PromoteLowerTo(d16, s0);
  auto s0_high_16 = hn::PromoteUpperTo(d16, s0);
  auto s1_low_16 = hn::PromoteLowerTo(d16, s1);
  auto s1_high_16 = hn::PromoteUpperTo(d16, s1);
  auto s2_low_16 = hn::PromoteLowerTo(d16, s2);
  auto s2_high_16 = hn::PromoteUpperTo(d16, s2);
  auto s3_low_16 = hn::PromoteLowerTo(d16, s3);
  auto s3_high_16 = hn::PromoteUpperTo(d16, s3);
  auto s4_low_16 = hn::PromoteLowerTo(d16, s4);
  auto s4_high_16 = hn::PromoteUpperTo(d16, s4);
  auto s5_low_16 = hn::PromoteLowerTo(d16, s5);
  auto s5_high_16 = hn::PromoteUpperTo(d16, s5);

  auto sum_low = hn::WidenMulPairwiseAdd(d32, s0_low_16, coeff0);
  sum_low = hn::SatWidenMulPairwiseAccumulate(d32, s1_low_16, coeff1, sum_low);
  sum_low = hn::SatWidenMulPairwiseAccumulate(d32, s2_low_16, coeff2, sum_low);
  sum_low = hn::SatWidenMulPairwiseAccumulate(d32, s3_low_16, coeff3, sum_low);
  sum_low = hn::SatWidenMulPairwiseAccumulate(d32, s4_low_16, coeff4, sum_low);
  sum_low = hn::SatWidenMulPairwiseAccumulate(d32, s5_low_16, coeff5, sum_low);

  auto sum_high = hn::WidenMulPairwiseAdd(d32, s0_high_16, coeff0);
  sum_high =
      hn::SatWidenMulPairwiseAccumulate(d32, s1_high_16, coeff1, sum_high);
  sum_high =
      hn::SatWidenMulPairwiseAccumulate(d32, s2_high_16, coeff2, sum_high);
  sum_high =
      hn::SatWidenMulPairwiseAccumulate(d32, s3_high_16, coeff3, sum_high);
  sum_high =
      hn::SatWidenMulPairwiseAccumulate(d32, s4_high_16, coeff4, sum_high);
  sum_high =
      hn::SatWidenMulPairwiseAccumulate(d32, s5_high_16, coeff5, sum_high);

  auto shifted_low = hn::ShiftRight<1>(sum_low);
  auto shifted_high = hn::ShiftRight<1>(sum_high);
  auto res_low_16 = hn::DemoteTo(d16_half, shifted_low);
  auto res_high_16 = hn::DemoteTo(d16_half, shifted_high);

  return hn::Combine(d16, res_high_16, res_low_16);
}

template <class D16, class V8, class V8_mask, class V16_coeff>
HWY_ATTR HWY_INLINE hn::VFromD<D16> ConvolveLowbdX12Tap(
    D16 d16, V8 data1, V8 data2, V8_mask mask1, V8_mask mask2, V8_mask mask3,
    V16_coeff coeff0, V16_coeff coeff1, V16_coeff coeff2, V16_coeff coeff3,
    V16_coeff coeff4, V16_coeff coeff5) {
  auto s0 = hn::TableLookupBytes(data1, mask1);
  auto s1 = hn::TableLookupBytes(data1, mask2);
  auto s2 = hn::TableLookupBytes(data1, mask3);
  auto s3 = hn::TableLookupBytes(data2, mask1);
  auto s4 = hn::TableLookupBytes(data2, mask2);
  auto s5 = hn::TableLookupBytes(data2, mask3);
  return ConvolveLowbdX12TapFromS(d16, s0, s1, s2, s3, s4, s5, coeff0, coeff1,
                                  coeff2, coeff3, coeff4, coeff5);
}

template <class D16, class V8, class V16_coeff>
HWY_ATTR HWY_INLINE hn::VFromD<D16> ConvolveLowbdX12TapShuffleFree(
    D16 d16, V8 data1, V8 data2, V16_coeff coeff0, V16_coeff coeff1,
    V16_coeff coeff2, V16_coeff coeff3, V16_coeff coeff4, V16_coeff coeff5) {
  // We only support AVX2 and AVX512, which have > 8 lanes of int16_t.
  static_assert(hn::MaxLanes(D16()) > 8, "Only AVX2 and AVX512 are supported");

  using D8_twice = hn::DFromV<V8>;
  const D8_twice d8_twice;

  auto mask1 = hn::LoadDup128(d8_twice, kFilt1);
  auto mask2 = hn::LoadDup128(d8_twice, kFilt2);
  auto mask3 = hn::LoadDup128(d8_twice, kFilt3);
  return ConvolveLowbdX12Tap(d16, data1, data2, mask1, mask2, mask3, coeff0,
                             coeff1, coeff2, coeff3, coeff4, coeff5);
}

// Vertical convolve helper
template <int taps_y, class D32, class V16, class V16_coeff>
HWY_ATTR HWY_INLINE hn::VFromD<D32> ConvolveVertical(
    D32 d32, const V16 *z_arr, const V16_coeff *coeffs_v,
    hn::VFromD<D32> round_const_y) {
  constexpr int num_coeffs = taps_y / 2;
  auto sum = hn::SatWidenMulPairwiseAccumulate(d32, z_arr[0], coeffs_v[0],
                                               round_const_y);
  for (int k = 1; k < num_coeffs; ++k) {
    sum = hn::SatWidenMulPairwiseAccumulate(d32, z_arr[k], coeffs_v[k], sum);
  }
  return sum;
}

template <class DI8>
HWY_ATTR HWY_INLINE void PrepareCoeffsH(DI8 di8, const int16_t *filter,
                                        int taps, hn::VFromD<DI8> *coeffs) {
  int start_idx = (taps == 12) ? 0 : (4 - taps / 2);
  int num_coeffs = taps / 2;
  for (int k = 0; k < num_coeffs; ++k) {
    auto c0 = static_cast<int8_t>(filter[start_idx + k * 2] >> 1);
    auto c1 = static_cast<int8_t>(filter[start_idx + k * 2 + 1] >> 1);
    HWY_ALIGN int8_t coeff_arr[16] = { c0, c1, c0, c1, c0, c1, c0, c1,
                                       c0, c1, c0, c1, c0, c1, c0, c1 };
    coeffs[k] = hn::LoadDup128(di8, coeff_arr);
  }
}

template <class D16>
HWY_ATTR HWY_INLINE void PrepareCoeffsV(D16 d16, const int16_t *filter,
                                        int taps, hn::VFromD<D16> *coeffs) {
  int start_idx = (taps == 12) ? 0 : (4 - taps / 2);
  int num_coeffs = taps / 2;
  for (int k = 0; k < num_coeffs; ++k) {
    int16_t c0 = filter[start_idx + k * 2];
    int16_t c1 = filter[start_idx + k * 2 + 1];
    HWY_ALIGN int16_t coeff_arr[8] = { c0, c1, c0, c1, c0, c1, c0, c1 };
    coeffs[k] = hn::LoadDup128(d16, coeff_arr);
  }
}

template <class D16>
HWY_ATTR HWY_INLINE void PrepareCoeffs12(D16 d16, const int16_t *filter,
                                         hn::VFromD<D16> *coeffs) {
  for (int k = 0; k < 6; ++k) {
    int16_t c0 = filter[k * 2];
    int16_t c1 = filter[k * 2 + 1];
    HWY_ALIGN int16_t coeff_arr[8] = { c0, c1, c0, c1, c0, c1, c0, c1 };
    coeffs[k] = hn::LoadDup128(d16, coeff_arr);
  }
}

template <class D, class V128>
HWY_ATTR HWY_INLINE hn::VFromD<D> Combine4(D d, V128 r0, V128 r1, V128 r2,
                                           V128 r3) {
  using DHalf = hn::Half<D>;
  const DHalf d_half;
  using DQuarter = hn::Half<DHalf>;

  auto r01 = hn::Combine(d_half, hn::BitCast(DQuarter(), r1),
                         hn::BitCast(DQuarter(), r0));
  auto r23 = hn::Combine(d_half, hn::BitCast(DQuarter(), r3),
                         hn::BitCast(DQuarter(), r2));
  return hn::Combine(d, r23, r01);
}

template <class D, class V128>
HWY_ATTR HWY_INLINE hn::VFromD<D> Combine2(D d, V128 r0, V128 r1) {
  using DHalf = hn::Half<D>;
  return hn::Combine(d, hn::BitCast(DHalf(), r1), hn::BitCast(DHalf(), r0));
}

template <int num_rows, class D, class D128>
HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAndCombine(D d, D128 d128,
                                                 const uint8_t *src,
                                                 int src_stride) {
  HWY_IF_CONSTEXPR(num_rows == 4) {
    auto r0 = hn::LoadU(d128, src + 0 * src_stride);
    auto r1 = hn::LoadU(d128, src + 1 * src_stride);
    auto r2 = hn::LoadU(d128, src + 2 * src_stride);
    auto r3 = hn::LoadU(d128, src + 3 * src_stride);
    return Combine4(d, r0, r1, r2, r3);
  }
  HWY_IF_CONSTEXPR(num_rows == 2) {
    auto r0 = hn::LoadU(d128, src + 0 * src_stride);
    auto r1 = hn::LoadU(d128, src + 1 * src_stride);
    return Combine2(d, r0, r1);
  }
  return hn::ResizeBitCast(d, hn::LoadU(d128, src));
}

template <int num_rows, class D, class D128>
HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAndCombineClamped(D d, D128 d128,
                                                        const uint8_t *src,
                                                        int src_stride,
                                                        int row_idx,
                                                        int max_rows) {
  HWY_IF_CONSTEXPR(num_rows == 4) {
    int r0 = row_idx;
    int r1 = std::min(row_idx + 1, max_rows - 1);
    int r2 = std::min(row_idx + 2, max_rows - 1);
    int r3 = std::min(row_idx + 3, max_rows - 1);

    auto v0 = hn::LoadU(d128, src + r0 * src_stride);
    auto v1 = hn::LoadU(d128, src + r1 * src_stride);
    auto v2 = hn::LoadU(d128, src + r2 * src_stride);
    auto v3 = hn::LoadU(d128, src + r3 * src_stride);
    return Combine4(d, v0, v1, v2, v3);
  }
  HWY_IF_CONSTEXPR(num_rows == 2) {
    int r0 = row_idx;
    int r1 = std::min(row_idx + 1, max_rows - 1);

    auto v0 = hn::LoadU(d128, src + r0 * src_stride);
    auto v1 = hn::LoadU(d128, src + r1 * src_stride);
    return Combine2(d, v0, v1);
  }
  return hn::ResizeBitCast(d, hn::LoadU(d128, src + row_idx * src_stride));
}

/// Helper function for the vertical convolve pass.
// Template parameters:
// - `taps_y_val`: Number of vertical filter taps (2, 4, 6, 8, or 12).
// - `h_ge_8`: Compile-time flag indicating if block height is >= 8. Allows the
//   compiler to fully unroll the output loop (limit is 4 if true, 2 if false).
// - `skip_strip_im`: Compile-time flag indicating if we should skip loading
//   from `strip_im_buf` and instead compute horizontal convolve on the fly.
// - `round_0_const`, `round_1_const`: Compile-time rounding offsets, enabling
//   optimized shift calculations when non-zero.
// - `limit_val`: Compile-time iteration limit (1 for h=2, 2 for h=4, 3 for h=6,
//    4 for h>=8).
// - `IdxTbl`: Target-specific Highway type for table lookup indices.
// - `ComputeHRowBlock`: Callable type for computing horizontal convolve on the
//   fly (used when `skip_strip_im` is true).
template <int taps_y_val, bool h_ge_8, bool skip_strip_im, int round_0_const,
          int round_1_const, int limit_val, class IdxTbl,
          class ComputeHRowBlock = IVec16 (*)(int)>
HWY_ATTR HWY_INLINE void ConvolveVerticalPass(
    const int16_t *strip_im_buf, uint8_t *dst, int dst_stride, int w, int h,
    const IVec16 *coeffs_v, IVec32 round_const_y, int round_1, int bits,
    IdxTbl idx_tbl, ComputeHRowBlock compute_h_row_block = nullptr) {
  constexpr int num_coeffs = taps_y_val / 2;
  constexpr int num_z = limit_val + num_coeffs - 1;
  constexpr int num_h = num_z / 2 + 1;

  hn::Half<decltype(int16xN_tag)> d16_16;
  const auto zero_32 = hn::Zero(int32xN_tag);

  auto round_and_store = [&](IVec32 sum, int y) {
    auto res_32_A = hn::ShiftRightSame(sum, round_1);
    HWY_IF_CONSTEXPR(round_0_const > 0 && round_1_const > 0) {
      constexpr int bits_const =
          FILTER_BITS * 2 - round_0_const - round_1_const;
      HWY_IF_CONSTEXPR(bits_const > 0) {
        res_32_A = hn::ShiftRightSame(res_32_A, bits_const);
      }
    }
    else {
      if (bits > 0) {
        res_32_A = hn::ShiftRightSame(res_32_A, bits);
      }
    }

    auto max_32 = hn::Max(res_32_A, zero_32);
    auto res_8_A =
        hn::DemoteTo(uint8x16_capped_tag, hn::BitCast(uint32xN_tag, max_32));

    if (w == 4) {
      uint8_t tmpA[16];
      hn::StoreU(hn::LowerHalf(uint8x8_capped_tag, res_8_A), uint8x8_capped_tag,
                 tmpA);
      hn::StoreU(hn::UpperHalf(uint8x8_capped_tag, res_8_A), uint8x8_capped_tag,
                 tmpA + 8);
      __builtin_memcpy(dst + (y + 0) * dst_stride, tmpA, 4);
      __builtin_memcpy(dst + (y + 1) * dst_stride, tmpA + 8, 4);
    } else {
      hn::StoreU(hn::LowerHalf(uint8x8_capped_tag, res_8_A), uint8x8_capped_tag,
                 dst + (y + 0) * dst_stride);
      hn::StoreU(hn::UpperHalf(uint8x8_capped_tag, res_8_A), uint8x8_capped_tag,
                 dst + (y + 1) * dst_stride);
    }
  };

  for (int y = 0; y < h; y += 8) {
    IVec16 H[num_h];
    for (int k = 0; k < num_h; ++k) {
      HWY_IF_CONSTEXPR(skip_strip_im) { H[k] = compute_h_row_block(y + k * 4); }
      else {
        H[k] = hn::LoadU(int16xN_tag, strip_im_buf + (y + k * 4) * 8);
      }
    }

    IVec16 Z[num_z];
    for (int z_idx = 0; z_idx < num_z; ++z_idx) {
      IVec16 L;
      if (z_idx % 2 == 0) {
        L = H[z_idx / 2];
      } else {
        L = hn::Combine(int16xN_tag, hn::LowerHalf(d16_16, H[z_idx / 2 + 1]),
                        hn::UpperHalf(d16_16, H[z_idx / 2]));
      }
      Z[z_idx] = hn::TableLookupLanes(L, idx_tbl);
    }

    for (int i = 0; i < limit_val; ++i) {
      auto sum = ConvolveVertical<taps_y_val>(int32xN_tag, &Z[i], coeffs_v,
                                              round_const_y);
      round_and_store(sum, y + 2 * i);
    }
  }
}

template <int taps_x_const = 0, int taps_y_const = 0, int round_0_const = 0,
          int round_1_const = 0>
HWY_ATTR HWY_INLINE void Convolve2DSRHwyImpl(
    const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w,
    int h, const InterpFilterParams *filter_params_x,
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
    const int subpel_y_qn, ConvolveParams *conv_params) {
  const int taps_x = taps_x_const > 0 ? taps_x_const : filter_params_x->taps;
  const int taps_y = taps_y_const > 0 ? taps_y_const : filter_params_y->taps;
  const int round_0 = round_0_const > 0 ? round_0_const : conv_params->round_0;
  const int round_1 = round_1_const > 0 ? round_1_const : conv_params->round_1;

  const bool is_taps_x_12 =
      (taps_x_const == 12) || (taps_x_const == 0 && taps_x == 12);
  const bool is_taps_y_12 =
      (taps_y_const == 12) || (taps_y_const == 0 && taps_y == 12);

  const int fo_vert = taps_y / 2 - 1;
  const int fo_horiz = taps_x / 2 - 1;

  constexpr int kNumRows = 4;
  HWY_ALIGN_MAX int16_t strip_im_buf[(MAX_SB_SIZE + MAX_FILTER_TAP + 8) * 8];

  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
      filter_params_y, subpel_y_qn & SUBPEL_MASK);

  IVec8 coeffs_h[6] = { hn::Zero(int8xN_tag) };
  IVec16 coeffs_h_12[6];
  if (is_taps_x_12) {
    PrepareCoeffsV(int16xN_tag, x_filter, taps_x, coeffs_h_12);
  } else {
    PrepareCoeffsH(int8xN_tag, x_filter, taps_x, coeffs_h);
  }

  UVec8 mask1, mask2, mask3, mask4;
  HWY_IF_CONSTEXPR(taps_x_const != 8 && taps_x_const != 12) {
    mask1 = hn::LoadDup128(uint8xN_tag, kFilt1);
    mask2 = hn::LoadDup128(uint8xN_tag, kFilt2);
    mask3 = hn::LoadDup128(uint8xN_tag, kFilt3);
    mask4 = hn::LoadDup128(uint8xN_tag, kFilt4);
  }

  const auto round_const_h = hn::Set(int16xN_tag, 1 << (round_0 - 2));

  const int im_h = h + taps_y - 1;
  const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;

  auto convolve_horizontal = [&](auto data_vec) {
    IVec16 res;
    HWY_IF_CONSTEXPR(taps_x_const == 2 || taps_x_const == 4 ||
                     taps_x_const == 6 || taps_x_const == 8) {
      res = ConvolveLowbdXShuffleFree<taps_x_const>(int16xN_tag, data_vec,
                                                    coeffs_h);
    }
    else HWY_IF_CONSTEXPR(taps_x_const == 0) {
      if (taps_x == 2) {
        res = ConvolveLowbdX<2>(int16xN_tag, data_vec, &mask1, coeffs_h);
      } else if (taps_x == 4) {
        const UVec8 masks[] = { mask1, mask2 };
        res = ConvolveLowbdX<4>(int16xN_tag, data_vec, masks, coeffs_h);
      } else if (taps_x == 6) {
        const UVec8 masks[] = { mask1, mask2, mask3 };
        res = ConvolveLowbdX<6>(int16xN_tag, data_vec, masks, coeffs_h);
      } else {
        const UVec8 masks[] = { mask1, mask2, mask3, mask4 };
        res = ConvolveLowbdX<8>(int16xN_tag, data_vec, masks, coeffs_h);
      }
    }
    return res;
  };

  for (int j = 0; j < w; j += 8) {
    const bool skip_strip_im_buf =
        (is_taps_y_12 || is_taps_x_12) ? false : (h == 4 || h == 8);

    if (!skip_strip_im_buf) {
      int i = 0;
      for (; i < (im_h & ~(kNumRows - 1)); i += kNumRows) {
        IVec16 res;
        if (is_taps_x_12) {
          auto data1 = LoadAndCombine<kNumRows>(
              uint8xN_tag, uint8x16_capped_tag,
              src_ptr + i * src_stride + j + 0, src_stride);
          auto data2 = LoadAndCombine<kNumRows>(
              uint8xN_tag, uint8x16_capped_tag,
              src_ptr + i * src_stride + j + 6, src_stride);
          HWY_IF_CONSTEXPR(taps_x_const == 12) {
            res = ConvolveLowbdX12TapShuffleFree(
                int16xN_tag, data1, data2, coeffs_h_12[0], coeffs_h_12[1],
                coeffs_h_12[2], coeffs_h_12[3], coeffs_h_12[4], coeffs_h_12[5]);
          }
          else {
            res = ConvolveLowbdX12Tap(int16xN_tag, data1, data2, mask1, mask2,
                                      mask3, coeffs_h_12[0], coeffs_h_12[1],
                                      coeffs_h_12[2], coeffs_h_12[3],
                                      coeffs_h_12[4], coeffs_h_12[5]);
          }
        } else {
          auto data = LoadAndCombine<kNumRows>(uint8xN_tag, uint8x16_capped_tag,
                                               src_ptr + i * src_stride + j,
                                               src_stride);
          res = convolve_horizontal(data);
        }

        auto shifted_res =
            hn::ShiftRightSame(hn::Add(res, round_const_h), round_0 - 1);
        hn::StoreU(shifted_res, int16xN_tag, strip_im_buf + i * 8);
      }
      for (; i < im_h; i += kNumRows) {
        IVec16 res;
        if (is_taps_x_12) {
          auto data1 = LoadAndCombineClamped<kNumRows>(
              uint8xN_tag, uint8x16_capped_tag, src_ptr + j + 0, src_stride, i,
              im_h);
          auto data2 = LoadAndCombineClamped<kNumRows>(
              uint8xN_tag, uint8x16_capped_tag, src_ptr + j + 6, src_stride, i,
              im_h);
          HWY_IF_CONSTEXPR(taps_x_const == 12) {
            res = ConvolveLowbdX12TapShuffleFree(
                int16xN_tag, data1, data2, coeffs_h_12[0], coeffs_h_12[1],
                coeffs_h_12[2], coeffs_h_12[3], coeffs_h_12[4], coeffs_h_12[5]);
          }
          else {
            res = ConvolveLowbdX12Tap(int16xN_tag, data1, data2, mask1, mask2,
                                      mask3, coeffs_h_12[0], coeffs_h_12[1],
                                      coeffs_h_12[2], coeffs_h_12[3],
                                      coeffs_h_12[4], coeffs_h_12[5]);
          }
        } else {
          auto data =
              LoadAndCombineClamped<kNumRows>(uint8xN_tag, uint8x16_capped_tag,
                                              src_ptr + j, src_stride, i, im_h);
          res = convolve_horizontal(data);
        }

        auto shifted_res =
            hn::ShiftRightSame(hn::Add(res, round_const_h), round_0 - 1);
        hn::StoreU(shifted_res, int16xN_tag, strip_im_buf + i * 8);
      }
    }

    const int bits = FILTER_BITS * 2 - round_0 - round_1;

    IVec16 coeffs_v[6];
    HWY_IF_CONSTEXPR(taps_y_const > 0) {
      PrepareCoeffsV(int16xN_tag, y_filter, taps_y_const, coeffs_v);
    }
    else {
      PrepareCoeffsV(int16xN_tag, y_filter, taps_y, coeffs_v);
    }

    const int round_const_y_val =
        ((1 << round_1) >> 1) + ((bits > 0) ? (1 << (bits - 1 + round_1)) : 0);
    const auto round_const_y = hn::Set(int32xN_tag, round_const_y_val);

    HWY_ALIGN_MAX static constexpr int16_t idx_arr[32] = {
      0, 8,  1, 9,  2,  10, 3,  11, 4,  12, 5,  13, 6,  14, 7,  15,
      8, 16, 9, 17, 10, 18, 11, 19, 12, 20, 13, 21, 14, 22, 15, 23
    };
    auto idx_tbl = hn::SetTableIndices(int16xN_tag, idx_arr);

    auto compute_h_row_block = [&](int row_idx) {
      auto data =
          LoadAndCombineClamped<4>(uint8xN_tag, uint8x16_capped_tag,
                                   src_ptr + j, src_stride, row_idx, im_h);
      auto res = convolve_horizontal(data);
      return hn::ShiftRightSame(hn::Add(res, round_const_h), round_0 - 1);
    };

    if (h == 4 && !is_taps_x_12) {
      HWY_IF_CONSTEXPR(taps_y_const > 0) {
        constexpr bool skip_strip_im = taps_y_const == 12 ? false : true;
        ConvolveVerticalPass<taps_y_const, false, skip_strip_im, round_0_const,
                             round_1_const, /*limit_val=*/2>(
            strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
            round_1, bits, idx_tbl, compute_h_row_block);
      }
      else {
        if (taps_y == 2) {
          ConvolveVerticalPass<2, false, true, round_0_const, round_1_const,
                               /*limit_val=*/2>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        } else if (taps_y == 4) {
          ConvolveVerticalPass<4, false, true, round_0_const, round_1_const,
                               /*limit_val=*/2>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        } else if (taps_y == 6) {
          ConvolveVerticalPass<6, false, true, round_0_const, round_1_const,
                               /*limit_val=*/2>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        } else if (taps_y == 8) {
          ConvolveVerticalPass<8, false, true, round_0_const, round_1_const,
                               /*limit_val=*/2>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        }
      }
    } else if (h == 8 && !is_taps_x_12) {
      HWY_IF_CONSTEXPR(taps_y_const > 0) {
        constexpr bool skip_strip_im = taps_y_const == 12 ? false : true;
        ConvolveVerticalPass<taps_y_const, true, skip_strip_im, round_0_const,
                             round_1_const, /*limit_val=*/4>(
            strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
            round_1, bits, idx_tbl, compute_h_row_block);
      }
      else {
        if (taps_y == 2) {
          ConvolveVerticalPass<2, true, true, round_0_const, round_1_const,
                               /*limit_val=*/4>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        } else if (taps_y == 4) {
          ConvolveVerticalPass<4, true, true, round_0_const, round_1_const,
                               /*limit_val=*/4>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        } else if (taps_y == 6) {
          ConvolveVerticalPass<6, true, true, round_0_const, round_1_const,
                               /*limit_val=*/4>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        } else if (taps_y == 8) {
          ConvolveVerticalPass<8, true, true, round_0_const, round_1_const,
                               /*limit_val=*/4>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl, compute_h_row_block);
        }
      }
    } else {
      if (h >= 8) {
        if (is_taps_y_12) {
          ConvolveVerticalPass<12, true, false, round_0_const, round_1_const,
                               /*limit_val=*/4>(
              strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
              round_1, bits, idx_tbl);
        } else {
          HWY_IF_CONSTEXPR(taps_y_const > 0) {
            ConvolveVerticalPass<taps_y_const, true, false, round_0_const,
                                 round_1_const, /*limit_val=*/4>(
                strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                round_const_y, round_1, bits, idx_tbl);
          }
          else {
            if (taps_y == 2) {
              ConvolveVerticalPass<2, true, false, round_0_const, round_1_const,
                                   /*limit_val=*/4>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            } else if (taps_y == 4) {
              ConvolveVerticalPass<4, true, false, round_0_const, round_1_const,
                                   /*limit_val=*/4>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            } else if (taps_y == 6) {
              ConvolveVerticalPass<6, true, false, round_0_const, round_1_const,
                                   /*limit_val=*/4>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            } else if (taps_y == 8) {
              ConvolveVerticalPass<8, true, false, round_0_const, round_1_const,
                                   /*limit_val=*/4>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            }
          }
        }
      } else {
        if (h == 2) {
          if (is_taps_y_12) {
            ConvolveVerticalPass<12, false, false, round_0_const, round_1_const,
                                 /*limit_val=*/1>(
                strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                round_const_y, round_1, bits, idx_tbl);
          } else {
            HWY_IF_CONSTEXPR(taps_y_const > 0) {
              ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
                                   round_1_const, /*limit_val=*/1>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            }
            else {
              if (taps_y == 2) {
                ConvolveVerticalPass<2, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/1>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 4) {
                ConvolveVerticalPass<4, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/1>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 6) {
                ConvolveVerticalPass<6, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/1>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 8) {
                ConvolveVerticalPass<8, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/1>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              }
            }
          }
        } else if (h == 6) {
          if (is_taps_y_12) {
            ConvolveVerticalPass<12, false, false, round_0_const, round_1_const,
                                 /*limit_val=*/3>(
                strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                round_const_y, round_1, bits, idx_tbl);
          } else {
            HWY_IF_CONSTEXPR(taps_y_const > 0) {
              ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
                                   round_1_const, /*limit_val=*/3>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            }
            else {
              if (taps_y == 2) {
                ConvolveVerticalPass<2, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/3>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 4) {
                ConvolveVerticalPass<4, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/3>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 6) {
                ConvolveVerticalPass<6, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/3>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 8) {
                ConvolveVerticalPass<8, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/3>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              }
            }
          }
        } else {
          if (is_taps_y_12) {
            ConvolveVerticalPass<12, false, false, round_0_const, round_1_const,
                                 /*limit_val=*/2>(
                strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                round_const_y, round_1, bits, idx_tbl);
          } else {
            HWY_IF_CONSTEXPR(taps_y_const > 0) {
              ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
                                   round_1_const, /*limit_val=*/2>(
                  strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                  round_const_y, round_1, bits, idx_tbl);
            }
            else {
              if (taps_y == 2) {
                ConvolveVerticalPass<2, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/2>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 4) {
                ConvolveVerticalPass<4, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/2>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 6) {
                ConvolveVerticalPass<6, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/2>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              } else if (taps_y == 8) {
                ConvolveVerticalPass<8, false, false, round_0_const,
                                     round_1_const, /*limit_val=*/2>(
                    strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
                    round_const_y, round_1, bits, idx_tbl);
              }
            }
          }
        }
      }
    }
  }
}

template <int tx, int ty, int r0, int r1>
HWY_ATTR void Convolve2DSRRun(const uint8_t *src, int src_stride, uint8_t *dst,
                              int dst_stride, int w, int h,
                              const InterpFilterParams *filter_params_x,
                              const InterpFilterParams *filter_params_y,
                              const int subpel_x_qn, const int subpel_y_qn,
                              ConvolveParams *conv_params) {
  Convolve2DSRHwyImpl<tx, ty, r0, r1>(src, src_stride, dst, dst_stride, w, h,
                                      filter_params_x, filter_params_y,
                                      subpel_x_qn, subpel_y_qn, conv_params);
}

HWY_ATTR inline void Convolve2DSR(const uint8_t *src, int src_stride,
                                  uint8_t *dst, int dst_stride, int w, int h,
                                  const InterpFilterParams *filter_params_x,
                                  const InterpFilterParams *filter_params_y,
                                  const int subpel_x_qn, const int subpel_y_qn,
                                  ConvolveParams *conv_params) {
  const int round_0 = conv_params->round_0;
  const int round_1 = conv_params->round_1;

  if (round_0 == 3 && round_1 == 11) {
    const int tap_x = get_filter_tap(filter_params_x, subpel_x_qn);
    const int tap_y = get_filter_tap(filter_params_y, subpel_y_qn);
    switch (tap_x) {
      case 2:
        switch (tap_y) {
          case 2:
            Convolve2DSRRun<2, 2, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 4:
            Convolve2DSRRun<2, 4, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 6:
            Convolve2DSRRun<2, 6, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 8:
            Convolve2DSRRun<2, 8, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 12:
            Convolve2DSRRun<2, 12, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
        }
        break;
      case 4:
        switch (tap_y) {
          case 2:
            Convolve2DSRRun<4, 2, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 4:
            Convolve2DSRRun<4, 4, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 6:
            Convolve2DSRRun<4, 6, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 8:
            Convolve2DSRRun<4, 8, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 12:
            Convolve2DSRRun<4, 12, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
        }
        break;
      case 6:
        switch (tap_y) {
          case 2:
            Convolve2DSRRun<6, 2, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 4:
            Convolve2DSRRun<6, 4, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 6:
            Convolve2DSRRun<6, 6, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 8:
            Convolve2DSRRun<6, 8, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 12:
            Convolve2DSRRun<6, 12, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
        }
        break;
      case 8:
        switch (tap_y) {
          case 2:
            Convolve2DSRRun<8, 2, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 4:
            Convolve2DSRRun<8, 4, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 6:
            Convolve2DSRRun<8, 6, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 8:
            Convolve2DSRRun<8, 8, 3, 11>(src, src_stride, dst, dst_stride, w, h,
                                         filter_params_x, filter_params_y,
                                         subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 12:
            Convolve2DSRRun<8, 12, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
        }
        break;
      case 12:
        switch (tap_y) {
          case 2:
            Convolve2DSRRun<12, 2, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 4:
            Convolve2DSRRun<12, 4, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 6:
            Convolve2DSRRun<12, 6, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 8:
            Convolve2DSRRun<12, 8, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
          case 12:
            Convolve2DSRRun<12, 12, 3, 11>(
                src, src_stride, dst, dst_stride, w, h, filter_params_x,
                filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
            return;
        }
        break;
    }
  }

  Convolve2DSRRun<0, 0, 0, 0>(src, src_stride, dst, dst_stride, w, h,
                              filter_params_x, filter_params_y, subpel_x_qn,
                              subpel_y_qn, conv_params);
}

}  // namespace HWY_NAMESPACE
}  // namespace

HWY_AFTER_NAMESPACE();

#define MAKE_CONVOLVE_2D_SR(suffix)                                            \
  extern "C" void av1_convolve_2d_sr_##suffix(                                 \
      const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, \
      int h, const InterpFilterParams *filter_params_x,                        \
      const InterpFilterParams *filter_params_y, const int subpel_x_qn,        \
      const int subpel_y_qn, ConvolveParams *conv_params);                     \
  HWY_ATTR void av1_convolve_2d_sr_##suffix(                                   \
      const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, \
      int h, const InterpFilterParams *filter_params_x,                        \
      const InterpFilterParams *filter_params_y, const int subpel_x_qn,        \
      const int subpel_y_qn, ConvolveParams *conv_params) {                    \
    if (w < 4) {                                                               \
      av1_convolve_2d_sr_avx2(src, src_stride, dst, dst_stride, w, h,          \
                              filter_params_x, filter_params_y, subpel_x_qn,   \
                              subpel_y_qn, conv_params);                       \
      return;                                                                  \
    }                                                                          \
    HWY_NAMESPACE::Convolve2DSR(src, src_stride, dst, dst_stride, w, h,        \
                                filter_params_x, filter_params_y, subpel_x_qn, \
                                subpel_y_qn, conv_params);                     \
  }

#endif  // AOM_AV1_COMMON_CONVOLVE_2D_SR_HWY_H_