Added SIMD code for intra_matrix (DIP) prediction. Add SIMD code for av1_dip_matrix_multiplication().
diff --git a/av1/av1.cmake b/av1/av1.cmake index 6859013..5ff0ed9 100644 --- a/av1/av1.cmake +++ b/av1/av1.cmake
@@ -113,6 +113,8 @@ APPEND AOM_AV1_COMMON_SOURCES "${AOM_ROOT}/av1/common/intra_matrix.c" "${AOM_ROOT}/av1/common/intra_matrix.h" "${AOM_ROOT}/av1/common/intra_dip.cc" "${AOM_ROOT}/av1/common/intra_dip.h") + list(APPEND AOM_AV1_COMMON_INTRIN_AVX2 + "${AOM_ROOT}/av1/common/x86/intra_matrix_avx2.c") endif() if(CONFIG_AV1_ENCODER)
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 6597d9b..1c9a2b4 100644 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -323,6 +323,12 @@ "uint8_t* weights, uint16_t *dst, ptrdiff_t stride, uint16_t* second_pred, ptrdiff_t second_stride, int bw, int bh"; } +# Data-driven intra prediction (DIP) +if (aom_config("CONFIG_DIP") eq "yes") { + add_proto qw/void av1_dip_matrix_multiplication/, "const uint16_t *A, const uint16_t *B, uint16_t *C, int bd"; + specialize qw/av1_dip_matrix_multiplication avx2/ +} + # build compound seg mask functions add_proto qw/void av1_build_compound_diffwtd_mask_highbd/, "uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint16_t *src0, int src0_stride, const uint16_t *src1, int src1_stride, int h, int w, int bd"; specialize qw/av1_build_compound_diffwtd_mask_highbd ssse3 avx2/;
diff --git a/av1/common/intra_matrix.c b/av1/common/intra_matrix.c index fbbfc2a..84ce187 100644 --- a/av1/common/intra_matrix.c +++ b/av1/common/intra_matrix.c
@@ -15,14 +15,8 @@ #include "intra_dip.h" #include "intra_matrix.h" -#define ROWS 64 -#define COLS 11 -#define BITS 12 -#define OFFSET (1 << (12 - 1)) -#define SCALE 4 - -static const uint16_t - av1_intra_matrix_weights[INTRA_DIP_MODE_CNT][ROWS][COLS] = { +const uint16_t + av1_intra_matrix_weights[INTRA_DIP_MODE_CNT][DIP_ROWS][DIP_COLS] = { { { 776, 1714, 1077, 998, 1043, 1437, 1157, 1027, 1027, 1011, 1023 }, { 845, 1519, 1598, 892, 1088, 988, 1219, 1079, 1039, 1000, 1022 }, @@ -426,17 +420,17 @@ // B - pointer to feature vector // C - 8x8 output prediction // bd - bit depth -static void matrix_multiplication(const uint16_t *A, const uint16_t *B, - uint16_t *C, int bd) { +void av1_dip_matrix_multiplication_c(const uint16_t *A, const uint16_t *B, + uint16_t *C, int bd) { int sum = 0; - for (int j = 0; j < COLS; j++) sum += B[j]; + for (int j = 0; j < DIP_FEATURES; j++) sum += B[j]; - for (int i = 0; i < ROWS; i++) { + for (int i = 0; i < DIP_ROWS; i++) { int c = 0; - for (int j = 0; j < COLS; j++) { - c += SCALE * A[i * COLS + j] * B[j]; + for (int j = 0; j < DIP_FEATURES; j++) { + c += DIP_SCALE * A[i * DIP_COLS + j] * B[j]; } - c = ((c + OFFSET) >> BITS) - sum; + c = ((c + DIP_OFFSET) >> DIP_BITS) - sum; C[i] = clip_pixel_highbd(c, bd); } } @@ -447,5 +441,5 @@ assert(mode >= 0 && mode < INTRA_DIP_MODE_CNT); const uint16_t *A = &av1_intra_matrix_weights[mode][0][0]; - matrix_multiplication(A, input, output, bd); + av1_dip_matrix_multiplication(A, input, output, bd); }
diff --git a/av1/common/intra_matrix.h b/av1/common/intra_matrix.h index ab83d92..c428186 100644 --- a/av1/common/intra_matrix.h +++ b/av1/common/intra_matrix.h
@@ -9,5 +9,14 @@ * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ +#define DIP_ROWS 64 +#define DIP_COLS 16 +#define DIP_BITS 12 +#define DIP_OFFSET (1 << (12 - 1)) +#define DIP_SCALE 4 +#define DIP_FEATURES 11 + +extern const uint16_t av1_intra_matrix_weights[][DIP_ROWS][DIP_COLS]; + void av1_intra_matrix_pred(const uint16_t *input, int mode, uint16_t *output, int bd);
diff --git a/av1/common/x86/intra_matrix_avx2.c b/av1/common/x86/intra_matrix_avx2.c new file mode 100644 index 0000000..bdcfdc3 --- /dev/null +++ b/av1/common/x86/intra_matrix_avx2.c
@@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include <immintrin.h> /* AVX2 */ + +#include "aom_dsp/aom_dsp_common.h" +#include "av1/common/intra_matrix.h" + +// Multiply 11 element feature vector with matrix to generate 8x8 prediction. +// A - pointer to matrix +// B - pointer to feature vector +// C - 8x8 output prediction +// bd - bit depth +void av1_dip_matrix_multiplication_avx2(const uint16_t *A, const uint16_t *B, + uint16_t *C, int bd) { + static const uint16_t mask[16] = { -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 0, 0, 0, 0, 0 }; + + __m256i in0 = _mm256_lddqu_si256((__m256i *)B); + __m256i in_mask = _mm256_lddqu_si256((__m256i *)mask); + in0 = _mm256_and_si256(in0, in_mask); + // in0 = { B0, B1, B2, B3, B4, B5, B6, B7 | B8, B9, B10, 0, 0, 0, 0, 0 } + __m256i negsum = _mm256_madd_epi16(in0, in_mask); + negsum = _mm256_hadd_epi32(negsum, negsum); + negsum = _mm256_hadd_epi32(negsum, negsum); + negsum = _mm256_slli_epi32(negsum, DIP_BITS - 2); + __m128i offset = _mm_set1_epi32(DIP_OFFSET >> 2); + __m128i maxval = _mm_set1_epi32((1 << bd) - 1); + __m128i zero = _mm_setzero_si128(); + + for (int i = 0; i < DIP_ROWS; i += 4) { + __m256i row0 = _mm256_lddqu_si256((__m256i *)&A[i * DIP_COLS]); + __m256i row1 = _mm256_lddqu_si256((__m256i *)&A[(i + 1) * DIP_COLS]); + __m256i row2 = _mm256_lddqu_si256((__m256i *)&A[(i + 2) * DIP_COLS]); + __m256i row3 = _mm256_lddqu_si256((__m256i *)&A[(i + 3) * DIP_COLS]); + __m256i m0 = _mm256_madd_epi16(row0, in0); + __m256i m1 = _mm256_madd_epi16(row1, in0); + __m256i m2 = _mm256_madd_epi16(row2, in0); + __m256i m3 = _mm256_madd_epi16(row3, in0); + __m256i m01 = _mm256_hadd_epi32(m0, m1); + __m256i m23 = _mm256_hadd_epi32(m2, m3); + __m256i m0123 = _mm256_hadd_epi32(m01, m23); + __m256i sum0 = _mm256_add_epi32(m0123, negsum); + __m128i sum0_lo = _mm256_castsi256_si128(sum0); + __m128i sum0_hi = _mm256_extracti128_si256(sum0, 1); + __m128i sum1 = _mm_add_epi32(sum0_lo, sum0_hi); + sum1 = _mm_add_epi32(sum1, offset); + sum1 = _mm_srai_epi32(sum1, DIP_BITS - 2); + sum1 = _mm_min_epi32(sum1, maxval); + sum1 = _mm_max_epi32(sum1, zero); + __m128i out0 = _mm_packus_epi32(sum1, sum1); + _mm_storeu_si64(&C[i], out0); + } +}
diff --git a/test/intra_matrix_test.cc b/test/intra_matrix_test.cc new file mode 100644 index 0000000..bdb3193 --- /dev/null +++ b/test/intra_matrix_test.cc
@@ -0,0 +1,133 @@ +/* + * Copyright (c) 2025, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 3-Clause Clear License + * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear + * License was not distributed with this source code in the LICENSE file, you + * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the + * Alliance for Open Media Patent License 1.0 was not distributed with this + * source code in the PATENTS file, you can obtain it at + * aomedia.org/license/patent-license/. + */ + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#include "third_party/googletest/src/googletest/include/gtest/gtest.h" +#include "test/register_state_check.h" +#include "test/function_equivalence_test.h" + +#include "config/aom_config.h" +#include "config/aom_dsp_rtcd.h" +#include "config/av1_rtcd.h" + +#include "aom/aom_integer.h" +#include "av1/common/enums.h" +#include "av1/common/intra_dip.h" +#include "av1/common/intra_matrix.h" + +using libaom_test::FunctionEquivalenceTest; + +namespace { + +template <typename F, typename T> +class IntraMatrixTest : public FunctionEquivalenceTest<F> { + protected: + static const int kIterations = 1000000; + static const int kBufSize = 8 * 8; + + virtual ~IntraMatrixTest() {} + + virtual void Execute(T *dip_tst) = 0; + + void Common() { + dip_ref_ = &dip_ref_data_[0]; + dip_tst_ = &dip_tst_data_[0]; + + Execute(dip_tst_); + + for (int r = 0; r < kBufSize; ++r) { + ASSERT_EQ(dip_ref_[r], dip_tst_[r]); + } + } + + T dip_arr_[DIP_ROWS * DIP_COLS]; + T dip_feat_[DIP_COLS]; + + T dip_ref_data_[kBufSize]; + T dip_tst_data_[kBufSize]; + + T *dip_ref_; + T *dip_tst_; +}; + +////////////////////////////////////////////////////////////////////////////// +// High bit-depth version +////////////////////////////////////////////////////////////////////////////// + +typedef void (*IMHB)(const uint16_t *A, const uint16_t *B, uint16_t *C, int bd); +typedef libaom_test::FuncParam<IMHB> IntraMatrixTestFuncsHBD; + +class IntraMatrixTestHB : public IntraMatrixTest<IMHB, uint16_t> { + protected: + void Execute(uint16_t *dip_tst) { + params_.ref_func(dip_arr_, dip_feat_, dip_ref_, bit_depth_); + ASM_REGISTER_STATE_CHECK( + params_.tst_func(dip_arr_, dip_feat_, dip_tst, bit_depth_)); + } + int bit_depth_; +}; + +TEST_P(IntraMatrixTestHB, RandomValues) { + for (int iter = 0; iter < kIterations && !HasFatalFailure(); ++iter) { + switch (rng_(3)) { + case 0: bit_depth_ = 8; break; + case 1: bit_depth_ = 10; break; + default: bit_depth_ = 12; break; + } + const int hi = 1 << bit_depth_; + + for (int i = 0; i < 16; ++i) { + dip_feat_[i] = rng_(hi); + } + int mode = iter % INTRA_DIP_MODE_CNT; + for (int r = 0; r < DIP_ROWS; ++r) { + for (int c = 0; c < DIP_FEATURES; ++c) { + dip_arr_[r * DIP_COLS + c] = av1_intra_matrix_weights[mode][r][c]; + } + } + + Common(); + } +} + +#if HAVE_AVX2 +INSTANTIATE_TEST_SUITE_P(AVX2, IntraMatrixTestHB, + ::testing::Values(IntraMatrixTestFuncsHBD( + av1_dip_matrix_multiplication_c, + av1_dip_matrix_multiplication_avx2))); +#endif // HAVE_AVX2 + +// Speed tests + +TEST_P(IntraMatrixTestHB, DISABLED_Speed) { + const int test_count = 10000000; + bit_depth_ = 12; + const int hi = 1 << bit_depth_; + for (int i = 0; i < 16; ++i) { + dip_feat_[i] = rng_(hi); + } + for (int r = 0; r < 64; ++r) { + for (int c = 0; c < 11; ++c) { + dip_arr_[r * 16 + c] = av1_intra_matrix_weights[0][r][c]; + } + } + dip_tst_ = &dip_tst_data_[0]; + for (int iter = 0; iter < test_count; ++iter) { + ASM_REGISTER_STATE_CHECK( + params_.tst_func(dip_arr_, dip_feat_, dip_tst_, bit_depth_)); + } +} + +} // namespace
diff --git a/test/test.cmake b/test/test.cmake index be7ff2b..33fb16d 100644 --- a/test/test.cmake +++ b/test/test.cmake
@@ -104,6 +104,11 @@ "${AOM_ROOT}/test/simd_cmp_impl.h" "${AOM_ROOT}/test/simd_impl.h") + if(CONFIG_DIP) + list(APPEND AOM_UNIT_TEST_COMMON_SOURCES + "${AOM_ROOT}/test/intra_matrix_test.cc") + endif() + if(CONFIG_ACCOUNTING) list(APPEND AOM_UNIT_TEST_COMMON_SOURCES "${AOM_ROOT}/test/accounting_test.cc")