riscv64/cdef: Add the CDEF optimization Change-Id: Ic9184eb4a58256fbbe204835593c3d07bc8564fa
diff --git a/av1/av1.cmake b/av1/av1.cmake index 4865962..b5e656c 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -445,6 +445,9 @@ list(APPEND AOM_AV1_COMMON_INTRIN_VSX "${AOM_ROOT}/av1/common/ppc/cfl_ppc.c") +list(APPEND AOM_AV1_COMMON_INTRIN_RVV + "${AOM_ROOT}/av1/common/riscv/cdef_block_rvv.c") + if(CONFIG_THREE_PASS) list(APPEND AOM_AV1_ENCODER_SOURCES "${AOM_ROOT}/av1/encoder/thirdpass.c" "${AOM_ROOT}/av1/encoder/thirdpass.h") @@ -822,6 +825,13 @@ endif() endif() + if(HAVE_RVV) + if(AOM_AV1_COMMON_INTRIN_RVV) + add_intrinsics_object_library("-march=rv64gcv" "rvv" "aom_av1_common" + "AOM_AV1_COMMON_INTRIN_RVV") + endif() + endif() + # Pass the new lib targets up to the parent scope instance of # $AOM_LIB_TARGETS. set(AOM_LIB_TARGETS ${AOM_LIB_TARGETS} PARENT_SCOPE)
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 13e1d31..cfde810 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -495,22 +495,22 @@ # structs as arguments, which makes the v256 type of the intrinsics # hard to support, so optimizations for this target are disabled. if ($opts{config} !~ /libs-x86-win32-vs.*/) { - specialize qw/cdef_find_dir sse4_1 avx2 neon/, "$ssse3_x86"; + specialize qw/cdef_find_dir sse4_1 avx2 neon rvv/, "$ssse3_x86"; specialize qw/cdef_find_dir_dual sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_8_0 sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_8_1 sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_8_2 sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_8_3 sse4_1 avx2 neon/, "$ssse3_x86"; + specialize qw/cdef_filter_8_0 sse4_1 avx2 neon rvv/, "$ssse3_x86"; + specialize qw/cdef_filter_8_1 sse4_1 avx2 neon rvv/, "$ssse3_x86"; + specialize qw/cdef_filter_8_2 sse4_1 avx2 neon rvv/, "$ssse3_x86"; + specialize qw/cdef_filter_8_3 sse4_1 avx2 neon rvv/, "$ssse3_x86"; - specialize qw/cdef_filter_16_0 sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_16_1 sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_16_2 sse4_1 avx2 neon/, "$ssse3_x86"; - specialize qw/cdef_filter_16_3 sse4_1 avx2 neon/, "$ssse3_x86"; + specialize qw/cdef_filter_16_0 sse4_1 avx2 neon rvv/, "$ssse3_x86"; + specialize qw/cdef_filter_16_1 sse4_1 avx2 neon rvv/, "$ssse3_x86"; + specialize qw/cdef_filter_16_2 sse4_1 avx2 neon rvv/, "$ssse3_x86"; + specialize qw/cdef_filter_16_3 sse4_1 avx2 neon rvv/, "$ssse3_x86"; - specialize qw/cdef_copy_rect8_8bit_to_16bit sse4_1 avx2 neon/, "$ssse3_x86"; + specialize qw/cdef_copy_rect8_8bit_to_16bit sse4_1 avx2 neon rvv/, "$ssse3_x86"; if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") { - specialize qw/cdef_copy_rect8_16bit_to_16bit sse4_1 avx2 neon/, "$ssse3_x86"; + specialize qw/cdef_copy_rect8_16bit_to_16bit sse4_1 avx2 neon rvv/, "$ssse3_x86"; } }
diff --git a/av1/common/riscv/cdef_block_rvv.c b/av1/common/riscv/cdef_block_rvv.c new file mode 100644 index 0000000..8ccfc2a --- /dev/null +++ b/av1/common/riscv/cdef_block_rvv.c
@@ -0,0 +1,1354 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved. + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <riscv_vector.h> + +#include "config/aom_config.h" +#include "config/av1_rtcd.h" +#include "av1/common/cdef_block.h" + +// partial A is a 16-bit vector of the form: +// [x8 x7 x6 x5 x4 x3 x2 x1] and partial B has the form: +// [0 y1 y2 y3 y4 y5 y6 y7]. +// This function computes (x1^2+y1^2)*C1 + (x2^2+y2^2)*C2 + ... +// (x7^2+y2^7)*C7 + (x8^2+0^2)*C8 where the C1..C8 constants are in const1 +// and const2. +static inline vuint32m1_t fold_mul_and_sum_rvv(vint16m1_t partiala, + vint16m1_t partialb, + vuint32m1_t const1, + vuint32m1_t const2) { + // Square and add the corresponding x and y values. + vint32m2_t cost = __riscv_vwmul_vv_i32m2(partiala, partiala, 8); + cost = __riscv_vwmacc_vv_i32m2(cost, partialb, partialb, 8); + + // Multiply by constant. + vuint32m2_t tmp1_u32m2 = __riscv_vreinterpret_v_i32m2_u32m2(cost); + vuint32m1_t cost_u32m1 = __riscv_vmul_vv_u32m1( + __riscv_vlmul_trunc_v_u32m2_u32m1(tmp1_u32m2), const1, 4); + tmp1_u32m2 = __riscv_vslidedown_vx_u32m2(tmp1_u32m2, 4, 8); + vuint32m1_t ret = __riscv_vmacc_vv_u32m1( + cost_u32m1, __riscv_vlmul_trunc_v_u32m2_u32m1(tmp1_u32m2), const2, 4); + return ret; +} + +// This function computes the cost along directions 4, 5, 6, 7. (4 is diagonal +// down-right, 6 is vertical). +// +// For each direction the lines are shifted so that we can perform a +// basic sum on each vector element. For example, direction 5 is "south by +// southeast", so we need to add the pixels along each line i below: +// +// 0 1 2 3 4 5 6 7 +// 0 1 2 3 4 5 6 7 +// 8 0 1 2 3 4 5 6 +// 8 0 1 2 3 4 5 6 +// 9 8 0 1 2 3 4 5 +// 9 8 0 1 2 3 4 5 +// 10 9 8 0 1 2 3 4 +// 10 9 8 0 1 2 3 4 +// +// For this to fit nicely in vectors, the lines need to be shifted like so: +// 0 1 2 3 4 5 6 7 +// 0 1 2 3 4 5 6 7 +// 8 0 1 2 3 4 5 6 +// 8 0 1 2 3 4 5 6 +// 9 8 0 1 2 3 4 5 +// 9 8 0 1 2 3 4 5 +// 10 9 8 0 1 2 3 4 +// 10 9 8 0 1 2 3 4 +// +// In this configuration we can now perform SIMD additions to get the cost +// along direction 5. Since this won't fit into a single 128-bit vector, we use +// two of them to compute each half of the new configuration, and pad the empty +// spaces with zeros. Similar shifting is done for other directions, except +// direction 6 which is straightforward as it's the vertical direction. +static vuint32m1_t compute_vert_directions_rvv( + vint16m1_t lines_0, vint16m1_t lines_1, vint16m1_t lines_2, + vint16m1_t lines_3, vint16m1_t lines_4, vint16m1_t lines_5, + vint16m1_t lines_6, vint16m1_t lines_7, uint32_t cost[4], size_t vl) { + size_t VL_SLIDE_DOWN = __riscv_vsetvl_e16m1(16); + vint16m1_t vec_zero_i16m1 = __riscv_vmv_v_x_i16m1(0, vl); + + // Partial sums for lines 0 and 1. + vint16m1_t partial4a = + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_0, (8 - 1), vl); + vint16m1_t tmp1_i16m1 = + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_1, (8 - 2), vl); + partial4a = __riscv_vadd_vv_i16m1(partial4a, tmp1_i16m1, vl); + vint16m1_t partial4b = __riscv_vslide1down_vx_i16m1(lines_0, 0, vl); + tmp1_i16m1 = __riscv_vslidedown_vx_i16m1(lines_1, 2, VL_SLIDE_DOWN); + partial4b = __riscv_vadd_vv_i16m1(partial4b, tmp1_i16m1, vl); + tmp1_i16m1 = __riscv_vadd_vv_i16m1(lines_0, lines_1, VL_SLIDE_DOWN); + vint16m1_t partial5a = + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 3), vl); + vint16m1_t partial5b = + __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 3, VL_SLIDE_DOWN); + vint16m1_t partial7a = + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 6), vl); + vint16m1_t partial7b = + __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 6, VL_SLIDE_DOWN); + vint16m1_t partial6 = __riscv_vmv_v_v_i16m1(tmp1_i16m1, vl); + + // Partial sums for lines 2 and 3. + tmp1_i16m1 = __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_2, (8 - 3), vl); + partial4a = __riscv_vadd_vv_i16m1(partial4a, tmp1_i16m1, vl); + tmp1_i16m1 = __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_3, (8 - 4), vl); + partial4a = __riscv_vadd_vv_i16m1(partial4a, tmp1_i16m1, vl); + tmp1_i16m1 = __riscv_vslidedown_vx_i16m1(lines_2, 3, VL_SLIDE_DOWN); + partial4b = __riscv_vadd_vv_i16m1(partial4b, tmp1_i16m1, vl); + tmp1_i16m1 = __riscv_vslidedown_vx_i16m1(lines_3, 4, VL_SLIDE_DOWN); + partial4b = __riscv_vadd_vv_i16m1(partial4b, tmp1_i16m1, vl); + tmp1_i16m1 = __riscv_vadd_vv_i16m1(lines_2, lines_3, VL_SLIDE_DOWN); + partial5a = __riscv_vadd_vv_i16m1( + partial5a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 4), vl), vl); + partial5b = __riscv_vadd_vv_i16m1( + partial5b, __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 4, VL_SLIDE_DOWN), vl); + partial7a = __riscv_vadd_vv_i16m1( + partial7a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 5), vl), vl); + partial7b = __riscv_vadd_vv_i16m1( + partial7b, __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 5, VL_SLIDE_DOWN), vl); + partial6 = __riscv_vadd_vv_i16m1(partial6, tmp1_i16m1, vl); + + // Partial sums for lines 4 and 5. + partial4a = __riscv_vadd_vv_i16m1( + partial4a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_4, (8 - 5), vl), vl); + partial4a = __riscv_vadd_vv_i16m1( + partial4a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_5, (8 - 6), vl), vl); + partial4b = __riscv_vadd_vv_i16m1( + partial4b, __riscv_vslidedown_vx_i16m1(lines_4, 5, VL_SLIDE_DOWN), vl); + partial4b = __riscv_vadd_vv_i16m1( + partial4b, __riscv_vslidedown_vx_i16m1(lines_5, 6, VL_SLIDE_DOWN), vl); + tmp1_i16m1 = __riscv_vadd_vv_i16m1(lines_4, lines_5, VL_SLIDE_DOWN); + partial5a = __riscv_vadd_vv_i16m1( + partial5a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 5), vl), vl); + partial5b = __riscv_vadd_vv_i16m1( + partial5b, __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 5, VL_SLIDE_DOWN), vl); + partial7a = __riscv_vadd_vv_i16m1( + partial7a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 4), vl), vl); + partial7b = __riscv_vadd_vv_i16m1( + partial7b, __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 4, VL_SLIDE_DOWN), vl); + partial6 = __riscv_vadd_vv_i16m1(partial6, tmp1_i16m1, vl); + + // Partial sums for lines 6 and 7. + partial4a = __riscv_vadd_vv_i16m1( + partial4a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_6, (8 - 7), vl), vl); + partial4a = __riscv_vadd_vv_i16m1(partial4a, lines_7, vl); + partial4b = __riscv_vadd_vv_i16m1( + partial4b, __riscv_vslidedown_vx_i16m1(lines_6, 7, VL_SLIDE_DOWN), vl); + tmp1_i16m1 = __riscv_vadd_vv_i16m1(lines_6, lines_7, VL_SLIDE_DOWN); + partial5a = __riscv_vadd_vv_i16m1( + partial5a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 6), vl), vl); + partial5b = __riscv_vadd_vv_i16m1( + partial5b, __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 6, VL_SLIDE_DOWN), vl); + partial7a = __riscv_vadd_vv_i16m1( + partial7a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, tmp1_i16m1, (8 - 3), vl), vl); + partial7b = __riscv_vadd_vv_i16m1( + partial7b, __riscv_vslidedown_vx_i16m1(tmp1_i16m1, 3, VL_SLIDE_DOWN), vl); + partial6 = __riscv_vadd_vv_i16m1(partial6, tmp1_i16m1, vl); + + // const0 = { 840, 420, 280, 210, } + vuint32m1_t const0 = __riscv_vmv_s_x_u32m1(210, 4); + const0 = __riscv_vslide1up_vx_u32m1(const0, 280, 4); + const0 = __riscv_vslide1up_vx_u32m1(const0, 420, 4); + const0 = __riscv_vslide1up_vx_u32m1(const0, 840, 4); + + // const1 = { 168, 140, 120, 105, } + vuint32m1_t const1 = __riscv_vmv_s_x_u32m1(105, 4); + const1 = __riscv_vslide1up_vx_u32m1(const1, 120, 4); + const1 = __riscv_vslide1up_vx_u32m1(const1, 140, 4); + const1 = __riscv_vslide1up_vx_u32m1(const1, 168, 4); + + // const2 = { 0, 0, 420, 210, } + vuint32m1_t const2 = __riscv_vmv_v_x_u32m1(0, 4); + const2 = __riscv_vslide1down_vx_u32m1(const2, 420, 4); + const2 = __riscv_vslide1down_vx_u32m1(const2, 210, 4); + + // const3 = { 140, 105, 105, 105, }; + vuint32m1_t const3 = __riscv_vmv_v_x_u32m1(105, 4); + const3 = __riscv_vslide1up_vx_u32m1(const3, 140, 4); + + // Compute costs in terms of partial sums. + vint32m2_t tmp1_i32m2 = __riscv_vwmul_vv_i32m2(partial6, partial6, vl); + vint32m2_t partial6_s32 = __riscv_vslidedown_vx_i32m2(tmp1_i32m2, 4, vl); + partial6_s32 = __riscv_vadd_vv_i32m2(partial6_s32, tmp1_i32m2, 4); + + // Reverse partial B. + // pattern = { 6, 5, 4, 3, 2, 1, 0, 7, }. + vuint32m1_t costs_0, costs_1, costs_2, costs_3; + static const uint16_t tab_u16[8] = { + 6, 5, 4, 3, 2, 1, 0, 7, + }; + vuint16m1_t index_u16m1 = __riscv_vle16_v_u16m1(tab_u16, 8); + vint16m1_t partial4b_rv = + __riscv_vrgather_vv_i16m1(partial4b, index_u16m1, 8); + costs_0 = fold_mul_and_sum_rvv(partial4a, partial4b_rv, const0, const1); + vuint32m1_t partial6_u32 = __riscv_vreinterpret_v_i32m1_u32m1( + __riscv_vlmul_trunc_v_i32m2_i32m1(partial6_s32)); + costs_2 = __riscv_vmul_vx_u32m1(partial6_u32, 105, 4); + vint16m1_t partial5b_rv = + __riscv_vrgather_vv_i16m1(partial5b, index_u16m1, 8); + costs_1 = fold_mul_and_sum_rvv(partial5a, partial5b_rv, const2, const3); + vint16m1_t partial7b_rv = + __riscv_vrgather_vv_i16m1(partial7b, index_u16m1, 8); + costs_3 = fold_mul_and_sum_rvv(partial7a, partial7b_rv, const2, const3); + + // combine values + vuint32m1_t vec_scalar_u32m1 = __riscv_vmv_s_x_u32m1(0, 1); + vuint32m1_t cost0_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_0, vec_scalar_u32m1, 4); + vuint32m1_t cost1_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_1, vec_scalar_u32m1, 4); + vuint32m1_t cost2_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_2, vec_scalar_u32m1, 4); + vuint32m1_t cost3_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_3, vec_scalar_u32m1, 4); + + vuint32m1_t cost47 = __riscv_vslideup_vx_u32m1(cost0_sum, cost1_sum, 1, 4); + cost47 = __riscv_vslideup_vx_u32m1(cost47, cost2_sum, 2, 4); + cost47 = __riscv_vslideup_vx_u32m1(cost47, cost3_sum, 3, 4); + __riscv_vse32_v_u32m1(&cost[0], cost47, 4); + return cost47; +} + +static inline vuint32m1_t fold_mul_and_sum_pairwise_rvv(vint16m1_t partiala, + vint16m1_t partialb, + vint16m1_t partialc, + vuint32m1_t const0) { + vuint16m1_t vid_u16m1 = __riscv_vid_v_u16m1(4); + vuint16m1_t index_u16m1 = __riscv_vsll_vx_u16m1(vid_u16m1, 1, 4); + vint16m1_t tmp_i16m1 = __riscv_vslide1down_vx_i16m1(partiala, 0, 8); + vint32m2_t partiala_i32m2 = __riscv_vwadd_vv_i32m2(partiala, tmp_i16m1, 8); + tmp_i16m1 = __riscv_vslide1down_vx_i16m1(partialb, 0, 8); + vint32m2_t partialb_i32m2 = __riscv_vwadd_vv_i32m2(partialb, tmp_i16m1, 8); + + tmp_i16m1 = __riscv_vslide1down_vx_i16m1(partialc, 0, 8); + vint32m2_t partialc_i32m2 = __riscv_vwadd_vv_i32m2(partialc, tmp_i16m1, 8); + partiala_i32m2 = __riscv_vmul_vv_i32m2(partiala_i32m2, partiala_i32m2, 8); + partialb_i32m2 = __riscv_vmul_vv_i32m2(partialb_i32m2, partialb_i32m2, 8); + vint32m1_t partialb_i32m1 = __riscv_vlmul_trunc_v_i32m2_i32m1( + __riscv_vrgatherei16_vv_i32m2(partialb_i32m2, index_u16m1, 4)); + partialc_i32m2 = __riscv_vmul_vv_i32m2(partialc_i32m2, partialc_i32m2, 8); + partiala_i32m2 = __riscv_vadd_vv_i32m2(partiala_i32m2, partialc_i32m2, 8); + vint32m1_t partiala_i32m1 = __riscv_vlmul_trunc_v_i32m2_i32m1( + __riscv_vrgatherei16_vv_i32m2(partiala_i32m2, index_u16m1, 4)); + + vuint32m1_t cost = __riscv_vmul_vx_u32m1( + __riscv_vreinterpret_v_i32m1_u32m1(partialb_i32m1), 105, 4); + cost = __riscv_vmacc_vv_u32m1( + cost, __riscv_vreinterpret_v_i32m1_u32m1(partiala_i32m1), const0, 4); + return cost; +} + +static inline vint32m1_t horizontal_add_4d_s16x8(vint16m1_t lines_0, + vint16m1_t lines_1, + vint16m1_t lines_2, + vint16m1_t lines_3) { + vint32m1_t vec_scalar_i32m1 = __riscv_vmv_s_x_i32m1(0, 1); + vint32m1_t lines0_sum = + __riscv_vwredsum_vs_i16m1_i32m1(lines_0, vec_scalar_i32m1, 8); + vint32m1_t lines1_sum = + __riscv_vwredsum_vs_i16m1_i32m1(lines_1, vec_scalar_i32m1, 8); + vint32m1_t lines2_sum = + __riscv_vwredsum_vs_i16m1_i32m1(lines_2, vec_scalar_i32m1, 8); + vint32m1_t lines3_sum = + __riscv_vwredsum_vs_i16m1_i32m1(lines_3, vec_scalar_i32m1, 8); + + vint32m1_t ret = __riscv_vslideup_vx_i32m1(lines0_sum, lines1_sum, 1, 4); + ret = __riscv_vslideup_vx_i32m1(ret, lines2_sum, 2, 4); + ret = __riscv_vslideup_vx_i32m1(ret, lines3_sum, 3, 4); + return ret; +} + +// This function computes the cost along directions 0, 1, 2, 3. (0 means +// 45-degree up-right, 2 is horizontal). +// +// For direction 1 and 3 ("east northeast" and "east southeast") the shifted +// lines need three vectors instead of two. For direction 1 for example, we need +// to compute the sums along the line i below: +// 0 0 1 1 2 2 3 3 +// 1 1 2 2 3 3 4 4 +// 2 2 3 3 4 4 5 5 +// 3 3 4 4 5 5 6 6 +// 4 4 5 5 6 6 7 7 +// 5 5 6 6 7 7 8 8 +// 6 6 7 7 8 8 9 9 +// 7 7 8 8 9 9 10 10 +// +// Which means we need the following configuration: +// 0 0 1 1 2 2 3 3 +// 1 1 2 2 3 3 4 4 +// 2 2 3 3 4 4 5 5 +// 3 3 4 4 5 5 6 6 +// 4 4 5 5 6 6 7 7 +// 5 5 6 6 7 7 8 8 +// 6 6 7 7 8 8 9 9 +// 7 7 8 8 9 9 10 10 +// +// Three vectors are needed to compute this, as well as some extra pairwise +// additions. +static vuint32m1_t compute_horiz_directions_rvv( + vint16m1_t lines_0, vint16m1_t lines_1, vint16m1_t lines_2, + vint16m1_t lines_3, vint16m1_t lines_4, vint16m1_t lines_5, + vint16m1_t lines_6, vint16m1_t lines_7, uint32_t cost[4], size_t vl) { + // Compute diagonal directions (1, 2, 3). + // Partial sums for lines 0 and 1. + size_t VL_SLIDE_DOWN = __riscv_vsetvl_e16m1(16); + vint16m1_t vec_zero_i16m1 = __riscv_vmv_v_x_i16m1(0, vl); + vint16m1_t partial0a = __riscv_vmv_v_v_i16m1(lines_0, vl); + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_1, (8 - 7), vl), vl); + vint16m1_t partial0b = __riscv_vslidedown_vx_i16m1(lines_1, 7, VL_SLIDE_DOWN); + vint16m1_t partial1a = __riscv_vadd_vv_i16m1( + lines_0, __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_1, (8 - 6), vl), + vl); + vint16m1_t partial1b = __riscv_vslidedown_vx_i16m1(lines_1, 6, VL_SLIDE_DOWN); + vint16m1_t partial3a = __riscv_vslidedown_vx_i16m1(lines_0, 2, VL_SLIDE_DOWN); + partial3a = __riscv_vadd_vv_i16m1( + partial3a, __riscv_vslidedown_vx_i16m1(lines_1, 4, VL_SLIDE_DOWN), vl); + vint16m1_t partial3b = + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_0, (8 - 2), vl); + partial3b = __riscv_vadd_vv_i16m1( + partial3b, __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_1, 4, vl), vl); + + // Partial sums for lines 2 and 3. + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_2, (8 - 6), vl), vl); + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_3, (8 - 5), vl), vl); + partial0b = __riscv_vadd_vv_i16m1( + partial0b, __riscv_vslidedown_vx_i16m1(lines_2, 6, VL_SLIDE_DOWN), vl); + partial0b = __riscv_vadd_vv_i16m1( + partial0b, __riscv_vslidedown_vx_i16m1(lines_3, 5, VL_SLIDE_DOWN), vl); + partial1a = __riscv_vadd_vv_i16m1( + partial1a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_2, (8 - 4), vl), vl); + partial1a = __riscv_vadd_vv_i16m1( + partial1a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_3, (8 - 2), vl), vl); + partial1b = __riscv_vadd_vv_i16m1( + partial1b, __riscv_vslidedown_vx_i16m1(lines_2, 4, VL_SLIDE_DOWN), vl); + partial1b = __riscv_vadd_vv_i16m1( + partial1b, __riscv_vslidedown_vx_i16m1(lines_3, 2, VL_SLIDE_DOWN), vl); + partial3a = __riscv_vadd_vv_i16m1( + partial3a, __riscv_vslidedown_vx_i16m1(lines_2, 6, VL_SLIDE_DOWN), vl); + partial3b = __riscv_vadd_vv_i16m1( + partial3b, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_2, (8 - 6), vl), vl); + partial3b = __riscv_vadd_vv_i16m1(partial3b, lines_3, vl); + + // Partial sums for lines 4 and 5. + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_4, (8 - 4), vl), vl); + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_5, (8 - 3), vl), vl); + partial0b = __riscv_vadd_vv_i16m1( + partial0b, __riscv_vslidedown_vx_i16m1(lines_4, 4, VL_SLIDE_DOWN), vl); + partial0b = __riscv_vadd_vv_i16m1( + partial0b, __riscv_vslidedown_vx_i16m1(lines_5, 3, VL_SLIDE_DOWN), vl); + partial1b = __riscv_vadd_vv_i16m1(partial1b, lines_4, vl); + partial1b = __riscv_vadd_vv_i16m1( + partial1b, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_5, (8 - 6), vl), vl); + vint16m1_t partial1c = __riscv_vslidedown_vx_i16m1(lines_5, 6, VL_SLIDE_DOWN); + partial3b = __riscv_vadd_vv_i16m1( + partial3b, __riscv_vslidedown_vx_i16m1(lines_4, 2, VL_SLIDE_DOWN), vl); + partial3b = __riscv_vadd_vv_i16m1( + partial3b, __riscv_vslidedown_vx_i16m1(lines_5, 4, VL_SLIDE_DOWN), vl); + vint16m1_t partial3c = + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_4, (8 - 2), vl); + partial3c = __riscv_vadd_vv_i16m1( + partial3c, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_5, (8 - 4), vl), vl); + + // Partial sums for lines 6 and 7. + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_6, (8 - 2), vl), vl); + partial0a = __riscv_vadd_vv_i16m1( + partial0a, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_7, (8 - 1), vl), vl); + partial0b = __riscv_vadd_vv_i16m1( + partial0b, __riscv_vslidedown_vx_i16m1(lines_6, 2, VL_SLIDE_DOWN), vl); + partial0b = __riscv_vadd_vv_i16m1( + partial0b, __riscv_vslide1down_vx_i16m1(lines_7, 0, vl), vl); + partial1b = __riscv_vadd_vv_i16m1( + partial1b, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_6, (8 - 4), vl), vl); + partial1b = __riscv_vadd_vv_i16m1( + partial1b, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_7, (8 - 2), vl), vl); + partial1c = __riscv_vadd_vv_i16m1( + partial1c, __riscv_vslidedown_vx_i16m1(lines_6, 4, VL_SLIDE_DOWN), vl); + partial1c = __riscv_vadd_vv_i16m1( + partial1c, __riscv_vslidedown_vx_i16m1(lines_7, 2, VL_SLIDE_DOWN), vl); + partial3b = __riscv_vadd_vv_i16m1( + partial3b, __riscv_vslidedown_vx_i16m1(lines_6, 6, VL_SLIDE_DOWN), vl); + partial3c = __riscv_vadd_vv_i16m1( + partial3c, + __riscv_vslideup_vx_i16m1(vec_zero_i16m1, lines_6, (8 - 6), vl), vl); + partial3c = __riscv_vadd_vv_i16m1(partial3c, lines_7, vl); + + // Special case for direction 2 as it's just a sum along each line. + vint32m1_t partial2a = + horizontal_add_4d_s16x8(lines_0, lines_1, lines_2, lines_3); + vint32m1_t partial2b = + horizontal_add_4d_s16x8(lines_4, lines_5, lines_6, lines_7); + vuint32m1_t partial2a_u32 = __riscv_vreinterpret_v_i32m1_u32m1( + __riscv_vmul_vv_i32m1(partial2a, partial2a, 4)); + vuint32m1_t partial2b_u32 = __riscv_vreinterpret_v_i32m1_u32m1( + __riscv_vmul_vv_i32m1(partial2b, partial2b, 4)); + + // const0 = { 840, 420, 280, 210, } + vuint32m1_t const0 = __riscv_vmv_s_x_u32m1(210, 4); + const0 = __riscv_vslide1up_vx_u32m1(const0, 280, 4); + const0 = __riscv_vslide1up_vx_u32m1(const0, 420, 4); + const0 = __riscv_vslide1up_vx_u32m1(const0, 840, 4); + + // const1 = { 168, 140, 120, 105, } + vuint32m1_t const1 = __riscv_vmv_s_x_u32m1(105, 4); + const1 = __riscv_vslide1up_vx_u32m1(const1, 120, 4); + const1 = __riscv_vslide1up_vx_u32m1(const1, 140, 4); + const1 = __riscv_vslide1up_vx_u32m1(const1, 168, 4); + + // const2 = { 420, 210, 140, 105, }; + vuint32m1_t const2 = __riscv_vmv_s_x_u32m1(105, 4); + const2 = __riscv_vslide1up_vx_u32m1(const2, 140, 4); + const2 = __riscv_vslide1up_vx_u32m1(const2, 210, 4); + const2 = __riscv_vslide1up_vx_u32m1(const2, 420, 4); + + static const uint16_t tab_u16[8] = { + 0, 6, 5, 4, 3, 2, 1, 0, + }; + vuint32m1_t costs_0, costs_1, costs_2, costs_3; + vuint16m1_t template_u16m1 = __riscv_vle16_v_u16m1(tab_u16, 8); + + // Reverse partial c. + // pattern = { 6, 5, 4, 3, 2, 1, 0, 7, } + vuint16m1_t index_u16m1 = __riscv_vslide1down_vx_u16m1(template_u16m1, 7, 8); + vint16m1_t partial0b_rv = + __riscv_vrgather_vv_i16m1(partial0b, index_u16m1, 8); + costs_0 = fold_mul_and_sum_rvv(partial0a, partial0b_rv, const0, const1); + + // Reverse partial c. + // pattern = { 5, 4, 3, 2, 1, 0, 6, 7, } + vuint16m1_t index_pair_u16m1 = + __riscv_vslide1down_vx_u16m1(template_u16m1, 6, 8); + index_pair_u16m1 = __riscv_vslide1down_vx_u16m1(index_pair_u16m1, 7, 8); + vint16m1_t partialc_rv = + __riscv_vrgather_vv_i16m1(partial1c, index_pair_u16m1, 8); + costs_1 = + fold_mul_and_sum_pairwise_rvv(partial1a, partial1b, partialc_rv, const2); + + costs_2 = __riscv_vadd_vv_u32m1(partial2a_u32, partial2b_u32, 4); + costs_2 = __riscv_vmul_vx_u32m1(costs_2, 105, 4); + + vint16m1_t partial3a_rv = + __riscv_vrgather_vv_i16m1(partial3a, index_pair_u16m1, 8); + costs_3 = + fold_mul_and_sum_pairwise_rvv(partial3c, partial3b, partial3a_rv, const2); + + // combine values + vuint32m1_t vec_scalar_u32m1 = __riscv_vmv_s_x_u32m1(0, 1); + vuint32m1_t cost0_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_0, vec_scalar_u32m1, 4); + vuint32m1_t cost1_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_1, vec_scalar_u32m1, 4); + vuint32m1_t cost2_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_2, vec_scalar_u32m1, 4); + vuint32m1_t cost3_sum = + __riscv_vredsum_vs_u32m1_u32m1(costs_3, vec_scalar_u32m1, 4); + + costs_0 = __riscv_vslideup_vx_u32m1(cost0_sum, cost1_sum, 1, 4); + costs_0 = __riscv_vslideup_vx_u32m1(costs_0, cost2_sum, 2, 4); + costs_0 = __riscv_vslideup_vx_u32m1(costs_0, cost3_sum, 3, 4); + __riscv_vse32_v_u32m1(&cost[0], costs_0, 4); + return costs_0; +} + +int cdef_find_dir_rvv(const uint16_t *img, int stride, int32_t *var, + int coeff_shift) { + size_t vl = 8; + size_t vlmax = __riscv_vsetvlmax_e16m1(); + vuint16m1_t s; + vint16m1_t lines_0, lines_1, lines_2, lines_3; + vint16m1_t lines_4, lines_5, lines_6, lines_7; + vuint16m1_t vec_zero_u16m1 = + __riscv_vmv_v_x_u16m1(0, __riscv_vsetvl_e16m1(16)); + + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_0 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_0 = __riscv_vsub_vx_i16m1(lines_0, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_1 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_1 = __riscv_vsub_vx_i16m1(lines_1, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_2 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_2 = __riscv_vsub_vx_i16m1(lines_2, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_3 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_3 = __riscv_vsub_vx_i16m1(lines_3, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_4 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_4 = __riscv_vsub_vx_i16m1(lines_4, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_5 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_5 = __riscv_vsub_vx_i16m1(lines_5, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_6 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_6 = __riscv_vsub_vx_i16m1(lines_6, 128, vl); + + img += stride; + if (vlmax == 8) + s = __riscv_vle16_v_u16m1(img, vl); + else + s = __riscv_vle16_v_u16m1_tu(vec_zero_u16m1, img, vl); + lines_7 = __riscv_vreinterpret_v_u16m1_i16m1( + __riscv_vsrl_vx_u16m1(s, coeff_shift, vl)); + lines_7 = __riscv_vsub_vx_i16m1(lines_7, 128, vl); + + // Compute "mostly vertical" directions. + uint32_t cost[8]; + vuint32m1_t cost47 = + compute_vert_directions_rvv(lines_0, lines_1, lines_2, lines_3, lines_4, + lines_5, lines_6, lines_7, cost + 4, vl); + + // Compute "mostly horizontal" directions. + vuint32m1_t cost03 = + compute_horiz_directions_rvv(lines_0, lines_1, lines_2, lines_3, lines_4, + lines_5, lines_6, lines_7, cost, vl); + + // Find max cost as well as its index to get best_dir. + // The max cost needs to be propagated in the whole vector to find its + // position in the original cost vectors cost03 and cost47. + vuint32m1_t vec_scalar_u32m1 = __riscv_vmv_s_x_u32m1(0, 1); + vuint32m1_t cost07 = __riscv_vmaxu_vv_u32m1(cost03, cost47, 4); + uint32_t best_cost = __riscv_vmv_x_s_u32m1_u32( + __riscv_vredmaxu_vs_u32m1_u32m1(cost07, vec_scalar_u32m1, 4)); + vbool32_t mask_cost = __riscv_vmseq_vx_u32m1_b32(cost03, best_cost, 4); + long best_dir = __riscv_vfirst_m_b32(mask_cost, 4); + if (best_dir == -1) { + mask_cost = __riscv_vmseq_vx_u32m1_b32(cost47, best_cost, 4); + best_dir = __riscv_vfirst_m_b32(mask_cost, 4); + best_dir += 4; + } + + // Difference between the optimal variance and the variance along the + // orthogonal direction. Again, the sum(x^2) terms cancel out. + *var = best_cost - cost[(best_dir + 4) & 7]; + + // We'd normally divide by 840, but dividing by 1024 is close enough + // for what we're going to do with this. + *var >>= 10; + return (int)best_dir; +} + +void cdef_copy_rect8_8bit_to_16bit_rvv(uint16_t *dst, int dstride, + const uint8_t *src, int sstride, + int width, int height) { + do { + int w = 0; + size_t num_cols = width; + while (num_cols > 0) { + size_t vl = __riscv_vsetvl_e8mf2(num_cols); + vuint8mf2_t u8_src = __riscv_vle8_v_u8mf2(src + w, vl); + vuint16m1_t u16_src = __riscv_vwcvtu_x_x_v_u16m1(u8_src, vl); + __riscv_vse16_v_u16m1(dst + w, u16_src, vl); + + w += vl; + num_cols -= vl; + } + src += sstride; + dst += dstride; + } while (--height != 0); +} + +void cdef_copy_rect8_16bit_to_16bit_rvv(uint16_t *dst, int dstride, + const uint16_t *src, int sstride, + int width, int height) { + do { + int w = 0; + size_t num_cols = width; + while (num_cols > 0) { + size_t vl = __riscv_vsetvl_e16m1(num_cols); + vuint16m1_t u16_src = __riscv_vle16_v_u16m1(src + w, vl); + __riscv_vse16_v_u16m1(dst + w, u16_src, vl); + + w += vl; + num_cols -= vl; + } + src += sstride; + dst += dstride; + } while (--height != 0); +} + +static inline vint16m1_t constrain16(vint16m1_t a, vint16m1_t b, + int16_t threshold, int16_t adjdamp, + size_t vl) { + if (!threshold) return __riscv_vmv_v_x_i16m1(0, vl); + const vbool16_t mask = __riscv_vmslt_vv_i16m1_b16(a, b, vl); + const vint16m1_t diff = __riscv_vsub_vv_i16m1(a, b, vl); + const vint16m1_t abs_diff = __riscv_vneg_v_i16m1_tumu(mask, diff, diff, vl); + const vint16m1_t shift = __riscv_vsra_vx_i16m1(abs_diff, adjdamp, vl); + const vint16m1_t thr = __riscv_vmv_v_x_i16m1(threshold, vl); + const vint16m1_t sub = __riscv_vsub_vv_i16m1(thr, shift, vl); + const vint16m1_t max = __riscv_vmax_vx_i16m1(sub, 0, vl); + const vint16m1_t min = __riscv_vmin_vv_i16m1(abs_diff, max, vl); + return __riscv_vneg_v_i16m1_tumu(mask, min, min, vl); +} + +static inline vint16m1_t vmax_mask(vint16m1_t a, vint16m1_t b, size_t vl) { + const vbool16_t mask = + __riscv_vmseq_vx_i16m1_b16(a, (int16_t)CDEF_VERY_LARGE, vl); + const vint16m1_t val = __riscv_vmerge_vvm_i16m1(a, b, mask, vl); + return __riscv_vmax_vv_i16m1(val, b, vl); +} + +static inline vint16m1_t load_strided_i16_4x2(int16_t *addr, + const ptrdiff_t stride, + size_t vl) { + const vint16m1_t px_l1 = __riscv_vle16_v_i16m1(addr + stride, vl); + const vint16m1_t px_l0 = __riscv_vle16_v_i16m1(addr, vl); + return __riscv_vslideup_vx_i16m1(px_l0, px_l1, 4, vl); +} + +static inline void store_strided_u8_4x2(uint8_t *addr, vuint8mf2_t vdst, + const ptrdiff_t stride, size_t vl) { + __riscv_vse8_v_u8mf2(addr, vdst, vl >> 1); + vdst = __riscv_vslidedown_vx_u8mf2(vdst, 4, vl); + __riscv_vse8_v_u8mf2(addr + stride, vdst, vl >> 1); +} + +static inline void store_strided_u16_4x2(uint16_t *addr, vuint16m1_t vdst, + const ptrdiff_t stride, size_t vl) { + __riscv_vse16_v_u16m1(addr, vdst, vl >> 1); + vdst = __riscv_vslidedown_vx_u16m1(vdst, 4, vl); + __riscv_vse16_v_u16m1(addr + stride, vdst, vl >> 1); +} + +#define LOAD_PIX(addr) \ + const vint16m1_t px = __riscv_vle16_v_i16m1((int16_t *)addr, vl); \ + vint16m1_t sum = __riscv_vmv_v_x_i16m1(0, vl) + +#define LOAD_PIX4(addr) \ + const vint16m1_t px = \ + load_strided_i16_4x2((int16_t *)addr, CDEF_BSTRIDE, vl); \ + vint16m1_t sum = __riscv_vmv_v_x_i16m1(0, vl) + +#define LOAD_DIR(p, addr, o0, o1) \ + const vint16m1_t p##0 = __riscv_vle16_v_i16m1((int16_t *)addr + o0, vl); \ + const vint16m1_t p##1 = __riscv_vle16_v_i16m1((int16_t *)addr - o0, vl); \ + const vint16m1_t p##2 = __riscv_vle16_v_i16m1((int16_t *)addr + o1, vl); \ + const vint16m1_t p##3 = __riscv_vle16_v_i16m1((int16_t *)addr - o1, vl) + +#define LOAD_DIR4(p, addr, o0, o1) \ + const vint16m1_t p##0 = \ + load_strided_i16_4x2((int16_t *)addr + o0, CDEF_BSTRIDE, vl); \ + const vint16m1_t p##1 = \ + load_strided_i16_4x2((int16_t *)addr - o0, CDEF_BSTRIDE, vl); \ + const vint16m1_t p##2 = \ + load_strided_i16_4x2((int16_t *)addr + o1, CDEF_BSTRIDE, vl); \ + const vint16m1_t p##3 = \ + load_strided_i16_4x2((int16_t *)addr - o1, CDEF_BSTRIDE, vl) + +#define MAKE_TAPS \ + const int *pri_taps = cdef_pri_taps[(pri_strength >> coeff_shift) & 1]; \ + const int16_t tap0 = (int16_t)(pri_taps[0]); \ + const int16_t tap1 = (int16_t)(pri_taps[1]) + +#define CONSTRAIN(p, strength, shift) \ + vint16m1_t p##_c0 = \ + constrain16(p##0, px, (int16_t)strength, (int16_t)shift, vl); \ + vint16m1_t p##_c1 = \ + constrain16(p##1, px, (int16_t)strength, (int16_t)shift, vl); \ + vint16m1_t p##_c2 = \ + constrain16(p##2, px, (int16_t)strength, (int16_t)shift, vl); \ + vint16m1_t p##_c3 = \ + constrain16(p##3, px, (int16_t)strength, (int16_t)shift, vl) + +#define SETUP_MINMAX \ + vint16m1_t max = px; \ + vint16m1_t min = px + +#define MIN_MAX(p) \ + do { \ + max = vmax_mask(p##0, max, vl); \ + min = __riscv_vmin_vv_i16m1(p##0, min, vl); \ + max = vmax_mask(p##1, max, vl); \ + min = __riscv_vmin_vv_i16m1(p##1, min, vl); \ + max = vmax_mask(p##2, max, vl); \ + min = __riscv_vmin_vv_i16m1(p##2, min, vl); \ + max = vmax_mask(p##3, max, vl); \ + min = __riscv_vmin_vv_i16m1(p##3, min, vl); \ + } while (0) + +#define PRI_0_UPDATE_SUM(p) \ + const vint16m1_t p##sum0 = __riscv_vadd_vv_i16m1(p##_c0, p##_c1, vl); \ + const vint16m1_t p##sum1 = __riscv_vadd_vv_i16m1(p##_c2, p##_c3, vl); \ + sum = __riscv_vmacc_vx_i16m1(sum, tap0, p##sum0, vl); \ + sum = __riscv_vmacc_vx_i16m1(sum, tap1, p##sum1, vl) + +#define UPDATE_SUM(p) \ + const vint16m1_t p##sum0 = __riscv_vadd_vv_i16m1(p##_c0, p##_c1, vl); \ + const vint16m1_t p##sum1 = __riscv_vadd_vv_i16m1(p##_c2, p##_c3, vl); \ + sum = __riscv_vadd_vv_i16m1(sum, p##sum0, vl); \ + sum = __riscv_vadd_vv_i16m1(sum, p##sum1, vl) + +#define SEC_0_UPDATE_SUM(p) \ + const vint16m1_t p##sum0 = __riscv_vadd_vv_i16m1(p##_c0, p##_c1, vl); \ + const vint16m1_t p##sum1 = __riscv_vadd_vv_i16m1(p##_c2, p##_c3, vl); \ + const vint16m1_t p##sum2 = __riscv_vadd_vv_i16m1(p##sum0, p##sum1, vl); \ + sum = __riscv_vadd_vv_i16m1(sum, __riscv_vsll_vx_i16m1(p##sum2, 1, vl), vl) + +#define BIAS \ + const vbool16_t mask = __riscv_vmslt_vx_i16m1_b16(sum, 0, vl); \ + const vint16m1_t v_8 = __riscv_vmv_v_x_i16m1(8, vl); \ + const vint16m1_t bias = __riscv_vsub_vx_i16m1_tumu(mask, v_8, v_8, 1, vl); \ + const vint16m1_t unclamped = __riscv_vadd_vv_i16m1( \ + px, __riscv_vsra_vx_i16m1(__riscv_vadd_vv_i16m1(bias, sum, vl), 4, vl), \ + vl) + +#define STORE4 \ + do { \ + store_strided_u8_4x2(dst8, vdst, dstride, vl); \ + \ + in += (CDEF_BSTRIDE << 1); \ + dst8 += (dstride << 1); \ + } while (0) + +#define STORE4_CLAMPED \ + do { \ + BIAS; \ + vint16m1_t clamped = __riscv_vmin_vv_i16m1( \ + __riscv_vmax_vv_i16m1(unclamped, min, vl), max, vl); \ + vuint8mf2_t vdst = __riscv_vncvt_x_x_w_u8mf2( \ + __riscv_vreinterpret_v_i16m1_u16m1(clamped), vl); \ + STORE4; \ + } while (0) + +#define STORE4_UNCLAMPED \ + do { \ + BIAS; \ + vuint8mf2_t vdst = __riscv_vncvt_x_x_w_u8mf2( \ + __riscv_vreinterpret_v_i16m1_u16m1(unclamped), vl); \ + STORE4; \ + } while (0) + +#define STORE8 \ + do { \ + __riscv_vse8_v_u8mf2(dst8, vdst, vl); \ + \ + in += CDEF_BSTRIDE; \ + dst8 += dstride; \ + } while (0) + +#define STORE8_CLAMPED \ + do { \ + BIAS; \ + vint16m1_t clamped = __riscv_vmin_vv_i16m1( \ + __riscv_vmax_vv_i16m1(unclamped, min, vl), max, vl); \ + vuint8mf2_t vdst = __riscv_vncvt_x_x_w_u8mf2( \ + __riscv_vreinterpret_v_i16m1_u16m1(clamped), vl); \ + STORE8; \ + } while (0) + +#define STORE8_UNCLAMPED \ + do { \ + BIAS; \ + vuint8mf2_t vdst = __riscv_vncvt_x_x_w_u8mf2( \ + __riscv_vreinterpret_v_i16m1_u16m1(unclamped), vl); \ + STORE8; \ + } while (0) + +#define STORE16_4 \ + do { \ + store_strided_u16_4x2(dst16, vdst, dstride, vl); \ + \ + in += (CDEF_BSTRIDE << 1); \ + dst16 += (dstride << 1); \ + } while (0) + +#define STORE16_4_CLAMPED \ + do { \ + BIAS; \ + vint16m1_t clamped = __riscv_vmin_vv_i16m1( \ + __riscv_vmax_vv_i16m1(unclamped, min, vl), max, vl); \ + vuint16m1_t vdst = __riscv_vreinterpret_v_i16m1_u16m1(clamped); \ + STORE16_4; \ + } while (0) + +#define STORE16_4_UNCLAMPED \ + do { \ + BIAS; \ + vuint16m1_t vdst = __riscv_vreinterpret_v_i16m1_u16m1(unclamped); \ + STORE16_4; \ + } while (0) + +#define STORE16 \ + do { \ + __riscv_vse16_v_u16m1(dst16, vdst, vl); \ + \ + in += CDEF_BSTRIDE; \ + dst16 += dstride; \ + } while (0) + +#define STORE16_CLAMPED \ + do { \ + BIAS; \ + vint16m1_t clamped = __riscv_vmin_vv_i16m1( \ + __riscv_vmax_vv_i16m1(unclamped, min, vl), max, vl); \ + vuint16m1_t vdst = __riscv_vreinterpret_v_i16m1_u16m1(clamped); \ + STORE16; \ + } while (0) + +#define STORE16_UNCLAMPED \ + do { \ + BIAS; \ + vuint16m1_t vdst = __riscv_vreinterpret_v_i16m1_u16m1(unclamped); \ + STORE16; \ + } while (0) + +void cdef_filter_8_0_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + const int po1 = cdef_directions[dir][0]; + const int po2 = cdef_directions[dir][1]; + const int s1o1 = cdef_directions[dir + 2][0]; + const int s1o2 = cdef_directions[dir + 2][1]; + const int s2o1 = cdef_directions[dir - 2][0]; + const int s2o2 = cdef_directions[dir - 2][1]; + MAKE_TAPS; + + if (pri_strength) { + pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength)); + } + if (sec_strength) { + sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength)); + } + + if (block_width == 8) { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + LOAD_PIX(in); + SETUP_MINMAX; + + // Primary pass + LOAD_DIR(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + MIN_MAX(p); + PRI_0_UPDATE_SUM(p); + + // Secondary pass 1 + LOAD_DIR(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + MIN_MAX(s); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + MIN_MAX(s2); + UPDATE_SUM(s2); + + // Store + STORE8_CLAMPED; + } while (--h != 0); + } else { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + LOAD_PIX4(in); + SETUP_MINMAX; + + // Primary pass + LOAD_DIR4(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + MIN_MAX(p); + PRI_0_UPDATE_SUM(p); + + // Secondary pass 1 + LOAD_DIR4(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + MIN_MAX(s); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR4(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + MIN_MAX(s2); + UPDATE_SUM(s2); + + // Store + STORE4_CLAMPED; + + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_8_1_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + (void)sec_strength; + (void)sec_damping; + + const int po1 = cdef_directions[dir][0]; + const int po2 = cdef_directions[dir][1]; + MAKE_TAPS; + + if (pri_strength) { + pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength)); + } + + if (block_width == 8) { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + LOAD_PIX(in); + + // Primary pass + LOAD_DIR(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + PRI_0_UPDATE_SUM(p); + + // Store + STORE8_UNCLAMPED; + } while (--h != 0); + } else { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + LOAD_PIX4(in); + + // Primary pass + LOAD_DIR4(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + PRI_0_UPDATE_SUM(p); + + // Store + STORE4_UNCLAMPED; + + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_8_2_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + (void)pri_strength; + (void)pri_damping; + (void)coeff_shift; + + const int s1o1 = cdef_directions[dir + 2][0]; + const int s1o2 = cdef_directions[dir + 2][1]; + const int s2o1 = cdef_directions[dir - 2][0]; + const int s2o2 = cdef_directions[dir - 2][1]; + + if (sec_strength) { + sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength)); + } + + if (block_width == 8) { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + LOAD_PIX(in); + + // Secondary pass 1 + LOAD_DIR(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + UPDATE_SUM(s2); + + // Store + STORE8_UNCLAMPED; + } while (--h != 0); + } else { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + LOAD_PIX4(in); + + // Secondary pass 1 + LOAD_DIR4(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR4(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + UPDATE_SUM(s2); + + // Store + STORE4_UNCLAMPED; + + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_8_3_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + (void)pri_strength; + (void)sec_strength; + (void)dir; + (void)pri_damping; + (void)sec_damping; + (void)coeff_shift; + + if (block_width == 8) { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + const vuint16m1_t px = __riscv_vle16_v_u16m1(in, vl); + const vuint8mf2_t vdst = __riscv_vncvt_x_x_w_u8mf2(px, vl); + __riscv_vse8_v_u8mf2(dst8, vdst, vl); + + in += CDEF_BSTRIDE; + dst8 += dstride; + } while (--h != 0); + } else { + uint8_t *dst8 = (uint8_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + const vint16m1_t px = + load_strided_i16_4x2((int16_t *)in, CDEF_BSTRIDE, vl); + vuint8mf2_t vdst = + __riscv_vncvt_x_x_w_u8mf2(__riscv_vreinterpret_v_i16m1_u16m1(px), vl); + store_strided_u8_4x2(dst8, vdst, dstride, vl); + + in += 2 * CDEF_BSTRIDE; + dst8 += 2 * dstride; + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_16_0_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + const int po1 = cdef_directions[dir][0]; + const int po2 = cdef_directions[dir][1]; + const int s1o1 = cdef_directions[dir + 2][0]; + const int s1o2 = cdef_directions[dir + 2][1]; + const int s2o1 = cdef_directions[dir - 2][0]; + const int s2o2 = cdef_directions[dir - 2][1]; + MAKE_TAPS; + + if (pri_strength) { + pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength)); + } + if (sec_strength) { + sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength)); + } + + if (block_width == 8) { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + LOAD_PIX(in); + SETUP_MINMAX; + + // Primary pass + LOAD_DIR(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + MIN_MAX(p); + PRI_0_UPDATE_SUM(p); + + // Secondary pass 1 + LOAD_DIR(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + MIN_MAX(s); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + MIN_MAX(s2); + UPDATE_SUM(s2); + + // Store + STORE16_CLAMPED; + } while (--h != 0); + } else { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + LOAD_PIX4(in); + SETUP_MINMAX; + + // Primary pass + LOAD_DIR4(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + MIN_MAX(p); + PRI_0_UPDATE_SUM(p); + + // Secondary pass 1 + LOAD_DIR4(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + MIN_MAX(s); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR4(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + MIN_MAX(s2); + UPDATE_SUM(s2); + + // Store + STORE16_4_CLAMPED; + + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_16_1_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + (void)sec_strength; + (void)sec_damping; + + const int po1 = cdef_directions[dir][0]; + const int po2 = cdef_directions[dir][1]; + MAKE_TAPS; + + if (pri_strength) { + pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength)); + } + + if (block_width == 8) { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + LOAD_PIX(in); + + // Primary pass + LOAD_DIR(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + PRI_0_UPDATE_SUM(p); + + // Store + STORE16_UNCLAMPED; + } while (--h != 0); + } else { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + LOAD_PIX4(in); + + // Primary pass + LOAD_DIR4(p, in, po1, po2); + CONSTRAIN(p, pri_strength, pri_damping); + PRI_0_UPDATE_SUM(p); + + // Store + STORE16_4_UNCLAMPED; + + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_16_2_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + (void)pri_strength; + (void)pri_damping; + (void)coeff_shift; + + const int s1o1 = cdef_directions[dir + 2][0]; + const int s1o2 = cdef_directions[dir + 2][1]; + const int s2o1 = cdef_directions[dir - 2][0]; + const int s2o2 = cdef_directions[dir - 2][1]; + + if (sec_strength) { + sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength)); + } + + if (block_width == 8) { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + LOAD_PIX(in); + + // Secondary pass 1 + LOAD_DIR(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + UPDATE_SUM(s2); + + // Store + STORE16_UNCLAMPED; + } while (--h != 0); + } else { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + LOAD_PIX4(in); + + // Secondary pass 1 + LOAD_DIR4(s, in, s1o1, s2o1); + CONSTRAIN(s, sec_strength, sec_damping); + SEC_0_UPDATE_SUM(s); + + // Secondary pass 2 + LOAD_DIR4(s2, in, s1o2, s2o2); + CONSTRAIN(s2, sec_strength, sec_damping); + UPDATE_SUM(s2); + + // Store + STORE16_4_UNCLAMPED; + + h -= 2; + } while (h != 0); + } +} + +void cdef_filter_16_3_rvv(void *dest, int dstride, const uint16_t *in, + int pri_strength, int sec_strength, int dir, + int pri_damping, int sec_damping, int coeff_shift, + int block_width, int block_height) { + (void)pri_strength; + (void)sec_strength; + (void)dir; + (void)pri_damping; + (void)sec_damping; + (void)coeff_shift; + + if (block_width == 8) { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width; + do { + const vuint16m1_t px = __riscv_vle16_v_u16m1(in, vl); + __riscv_vse16_v_u16m1(dst16, px, vl); + + in += CDEF_BSTRIDE; + dst16 += dstride; + } while (--h != 0); + } else { + uint16_t *dst16 = (uint16_t *)dest; + + int h = block_height; + const size_t vl = block_width << 1; + do { + const vint16m1_t px = + load_strided_i16_4x2((int16_t *)in, CDEF_BSTRIDE, vl); + vuint16m1_t vdst = __riscv_vreinterpret_v_i16m1_u16m1(px); + store_strided_u16_4x2(dst16, vdst, dstride, vl); + + in += 2 * CDEF_BSTRIDE; + dst16 += 2 * dstride; + h -= 2; + } while (h != 0); + } +}
diff --git a/test/cdef_test.cc b/test/cdef_test.cc index 410c9af..958e6e5 100644 --- a/test/cdef_test.cc +++ b/test/cdef_test.cc
@@ -618,7 +618,8 @@ using std::make_tuple; -#if ((AOM_ARCH_X86 && HAVE_SSSE3) || HAVE_SSE4_1 || HAVE_AVX2 || HAVE_NEON) +#if ((AOM_ARCH_X86 && HAVE_SSSE3) || HAVE_SSE4_1 || HAVE_AVX2 || HAVE_NEON || \ + HAVE_RVV) static const CdefFilterBlockFunctions kCdefFilterFuncC[] = { { &cdef_filter_8_0_c, &cdef_filter_8_1_c, &cdef_filter_8_2_c, &cdef_filter_8_3_c } @@ -811,6 +812,46 @@ #endif // CONFIG_AV1_HIGHBITDEPTH #endif +#if HAVE_RVV +static const CdefFilterBlockFunctions kCdefFilterFuncRvv[] = { + { &cdef_filter_8_0_rvv, &cdef_filter_8_1_rvv, &cdef_filter_8_2_rvv, + &cdef_filter_8_3_rvv } +}; + +static const CdefFilterBlockFunctions kCdefFilterHighbdFuncRvv[] = { + { &cdef_filter_16_0_rvv, &cdef_filter_16_1_rvv, &cdef_filter_16_2_rvv, + &cdef_filter_16_3_rvv } +}; + +INSTANTIATE_TEST_SUITE_P( + RVV, CDEFBlockTest, + ::testing::Combine(::testing::ValuesIn(kCdefFilterFuncRvv), + ::testing::ValuesIn(kCdefFilterFuncC), + ::testing::Values(BLOCK_4X4, BLOCK_4X8, BLOCK_8X4, + BLOCK_8X8), + ::testing::Range(0, 16), ::testing::Values(8))); +INSTANTIATE_TEST_SUITE_P( + RVV, CDEFBlockHighbdTest, + ::testing::Combine(::testing::ValuesIn(kCdefFilterHighbdFuncRvv), + ::testing::ValuesIn(kCdefFilterHighbdFuncC), + ::testing::Values(BLOCK_4X4, BLOCK_4X8, BLOCK_8X4, + BLOCK_8X8), + ::testing::Range(0, 16), ::testing::Range(10, 13, 2))); +INSTANTIATE_TEST_SUITE_P(RVV, CDEFFindDirTest, + ::testing::Values(make_tuple(&cdef_find_dir_rvv, + &cdef_find_dir_c))); + +INSTANTIATE_TEST_SUITE_P( + RVV, CDEFCopyRect8to16Test, + ::testing::Values(make_tuple(&cdef_copy_rect8_8bit_to_16bit_c, + &cdef_copy_rect8_8bit_to_16bit_rvv))); + +INSTANTIATE_TEST_SUITE_P( + RVV, CDEFCopyRect16to16Test, + ::testing::Values(make_tuple(&cdef_copy_rect8_16bit_to_16bit_c, + &cdef_copy_rect8_16bit_to_16bit_rvv))); +#endif + // Test speed for all supported architectures #if AOM_ARCH_X86 && HAVE_SSSE3 INSTANTIATE_TEST_SUITE_P( @@ -905,4 +946,24 @@ &cdef_find_dir_dual_c))); #endif +#if HAVE_RVV +INSTANTIATE_TEST_SUITE_P( + RVV, CDEFSpeedTest, + ::testing::Combine(::testing::ValuesIn(kCdefFilterFuncRvv), + ::testing::ValuesIn(kCdefFilterFuncC), + ::testing::Values(BLOCK_4X4, BLOCK_4X8, BLOCK_8X4, + BLOCK_8X8), + ::testing::Range(0, 16), ::testing::Values(8))); +INSTANTIATE_TEST_SUITE_P( + RVV, CDEFSpeedHighbdTest, + ::testing::Combine(::testing::ValuesIn(kCdefFilterHighbdFuncRvv), + ::testing::ValuesIn(kCdefFilterHighbdFuncC), + ::testing::Values(BLOCK_4X4, BLOCK_4X8, BLOCK_8X4, + BLOCK_8X8), + ::testing::Range(0, 16), ::testing::Values(10))); +INSTANTIATE_TEST_SUITE_P(RVV, CDEFFindDirSpeedTest, + ::testing::Values(make_tuple(&cdef_find_dir_rvv, + &cdef_find_dir_c))); +#endif + } // namespace